From a5027b5a60f699793318601ab4f22621f5727998 Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Mon, 18 Nov 2024 10:48:57 +0000 Subject: [PATCH 01/33] Revert [b11c0b7e61], for testing purposes only. --- win/Makefile.in | 9 +++++++-- win/makefile.vc | 9 +++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/win/Makefile.in b/win/Makefile.in index bb18d4f1637..35e63947e4b 100644 --- a/win/Makefile.in +++ b/win/Makefile.in @@ -549,8 +549,13 @@ ${TCL_ZIP_FILE}: ${ZIP_INSTALL_OBJS} ${DDE_DLL_FILE} ${REG_DLL_FILE} @( \ $(COPY) -a $(TOP_DIR)/library/* ${TCL_VFS_PATH}; \ $(COPY) -a ${TCL_VFS_PATH}/manifest.txt ${TCL_VFS_PATH}/pkgIndex.tcl; \ - rm -rf ${TCL_VFS_PATH}/dde; \ - rm -rf ${TCL_VFS_PATH}/registry; \ + if test "${ZIPFS_BUILD}" != "2" ; then \ + $(COPY) ${DDE_DLL_FILE} ${TCL_VFS_PATH}/dde; \ + $(COPY) ${REG_DLL_FILE} ${TCL_VFS_PATH}/registry; \ + else \ + rm -rf ${TCL_VFS_PATH}/dde; \ + rm -rf ${TCL_VFS_PATH}/registry; \ + fi \ ) (zip=`(realpath '${NATIVE_ZIP}' || readlink -m '${NATIVE_ZIP}') 2>/dev/null || \ (echo '${NATIVE_ZIP}' | sed "s?^\./?$$(pwd)/?")`; \ diff --git a/win/makefile.vc b/win/makefile.vc index 175c4b26129..951ff39961e 100644 --- a/win/makefile.vc +++ b/win/makefile.vc @@ -661,11 +661,16 @@ $(TCLSCRIPTZIP): $(TCLLIB) $(TCLSH) dlls @$(MKDIR) "$(LIBTCLVFS)" @$(CPYDIR) $(LIBDIR) "$(LIBTCLVFS)\tcl_library" @move /y "$(LIBTCLVFS)\tcl_library\manifest.txt" "$(LIBTCLVFS)\tcl_library\pkgIndex.tcl" > NUL +!if $(STATIC_BUILD) # Remove the registry and dde directories as the DLLS are still external @del "$(LIBTCLVFS)\tcl_library\registry\pkgIndex.tcl" @rmdir "$(LIBTCLVFS)\tcl_library\registry" @del "$(LIBTCLVFS)\tcl_library\dde\pkgIndex.tcl" @rmdir "$(LIBTCLVFS)\tcl_library\dde" +!else + @$(COPY) $(TCLDDELIB) "$(LIBTCLVFS)\tcl_library\dde + @$(COPY) $(TCLREGLIB) "$(LIBTCLVFS)\tcl_library\registry +!endif @echo cd {$(OUT_DIR)} > "$(OUT_DIR)\zipper.tcl" @echo file delete -force {$(@F)} >> "$(OUT_DIR)\zipper.tcl" @echo zipfs mkzip {$(@F)} {$(LIBTCLVFSSUBDIR)} {$(LIBTCLVFSSUBDIR)} >> "$(OUT_DIR)\zipper.tcl" @@ -1124,13 +1129,17 @@ install-libraries: tclConfig tcl-nmake install-msgs install-tzdata "$(MODULE_INSTALL_DIR)\9.0\platform\shell-$(PKG_SHELL_VER).tm" !endif @echo Installing $(TCLDDELIBNAME) +!if !$(STATIC_BUILD) @$(CPY) "$(TCLDDELIB)" "$(LIB_INSTALL_DIR)\dde$(DDEDOTVERSION)\" @$(CPY) "$(ROOT)\library\dde\pkgIndex.tcl" \ "$(LIB_INSTALL_DIR)\dde$(DDEDOTVERSION)\" +!endif @echo Installing $(TCLREGLIBNAME) +!if !$(STATIC_BUILD) @$(CPY) "$(TCLREGLIB)" "$(LIB_INSTALL_DIR)\registry$(REGDOTVERSION)\" @$(CPY) "$(ROOT)\library\registry\pkgIndex.tcl" \ "$(LIB_INSTALL_DIR)\registry$(REGDOTVERSION)\" +!endif !if !$(TCL_EMBED_SCRIPTS) @echo Installing encodings @$(CPY) "$(ROOT)\library\encoding\*.enc" \ From b75e24f0e660488ccf54a45dac0ec78ce689eb29 Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Mon, 18 Nov 2024 15:11:52 +0000 Subject: [PATCH 02/33] Fix for [a8e4f76ce4]: load library (dll) from zipfs-library causes a leak in temporary folder --- generic/tclIOUtil.c | 12 +- generic/tclInt.h | 2 +- generic/tclLoadNone.c | 2 +- unix/tclLoadDyld.c | 2 +- win/Makefile.in | 6 +- win/MemoryModule.c | 758 ++++++++++++++++++++++++++++++++++++++++++ win/MemoryModule.h | 109 ++++++ win/makefile.vc | 6 +- win/tclWinLoad.c | 70 ++++ 9 files changed, 953 insertions(+), 14 deletions(-) create mode 100644 win/MemoryModule.c create mode 100644 win/MemoryModule.h diff --git a/generic/tclIOUtil.c b/generic/tclIOUtil.c index 26d730304be..b2cfd8a5b7b 100644 --- a/generic/tclIOUtil.c +++ b/generic/tclIOUtil.c @@ -3229,7 +3229,8 @@ Tcl_LoadFile( */ { - int ret, size; + int ret; + size_t size; void *buffer; Tcl_StatBuf statBuf; Tcl_Channel data; @@ -3238,15 +3239,8 @@ Tcl_LoadFile( if (ret < 0) { goto mustCopyToTempAnyway; } - size = (int) statBuf.st_size; + size = statBuf.st_size; - /* - * Tcl_Read takes an int: Determine whether the file size is wide. - */ - - if (size != (Tcl_WideInt) statBuf.st_size) { - goto mustCopyToTempAnyway; - } data = Tcl_FSOpenFileChannel(interp, pathPtr, "rb", 0666); if (!data) { goto mustCopyToTempAnyway; diff --git a/generic/tclInt.h b/generic/tclInt.h index 38de84ad480..d85ccf82b24 100644 --- a/generic/tclInt.h +++ b/generic/tclInt.h @@ -3668,7 +3668,7 @@ MODULE_SCOPE int TclpDlopen(Tcl_Interp *interp, Tcl_Obj *pathPtr, Tcl_FSUnloadFileProc **unloadProcPtr, int flags); MODULE_SCOPE int TclpUtime(Tcl_Obj *pathPtr, struct utimbuf *tval); #ifdef TCL_LOAD_FROM_MEMORY -MODULE_SCOPE void * TclpLoadMemoryGetBuffer(Tcl_Interp *interp, int size); +MODULE_SCOPE void * TclpLoadMemoryGetBuffer(Tcl_Interp *interp, size_t size); MODULE_SCOPE int TclpLoadMemory(Tcl_Interp *interp, void *buffer, int size, int codeSize, Tcl_LoadHandle *loadHandle, Tcl_FSUnloadFileProc **unloadProcPtr, int flags); diff --git a/generic/tclLoadNone.c b/generic/tclLoadNone.c index f60f843b2dd..182aa86f1ed 100644 --- a/generic/tclLoadNone.c +++ b/generic/tclLoadNone.c @@ -64,7 +64,7 @@ TclpDlopen( MODULE_SCOPE void * TclpLoadMemoryGetBuffer( TCL_UNUSED(Tcl_Interp *), - TCL_UNUSED(int)) + TCL_UNUSED(size_t)) { return NULL; } diff --git a/unix/tclLoadDyld.c b/unix/tclLoadDyld.c index 2fdfabebab0..accfebde906 100644 --- a/unix/tclLoadDyld.c +++ b/unix/tclLoadDyld.c @@ -497,7 +497,7 @@ UnloadFile( MODULE_SCOPE void * TclpLoadMemoryGetBuffer( TCL_UNUSED(Tcl_Interp *), - int size) /* Size of desired buffer. */ + size_t size) /* Size of desired buffer. */ { void *buffer = NULL; diff --git a/win/Makefile.in b/win/Makefile.in index 35e63947e4b..8654904003d 100644 --- a/win/Makefile.in +++ b/win/Makefile.in @@ -265,7 +265,7 @@ MINIZIP_OBJS = \ ZIP_INSTALL_OBJS = @ZIP_INSTALL_OBJS@ CC_SWITCHES = -I"${BUILD_DIR}" -I"${GENERIC_DIR_NATIVE}" -I"${TOMMATH_DIR_NATIVE}" \ --I"${ZLIB_DIR_NATIVE}" -I"${WIN_DIR_NATIVE}" \ +-I"${ZLIB_DIR_NATIVE}" -I"${WIN_DIR_NATIVE}" -DTCL_LOAD_FROM_MEMORY \ ${CFLAGS} ${CFLAGS_WARNING} ${SHLIB_CFLAGS} -DMP_PREC=4 \ ${AC_FLAGS} ${COMPILE_DEBUG_FLAGS} ${NO_DEPRECATED_FLAGS} @@ -453,6 +453,7 @@ TOMMATH_OBJS = \ WIN_OBJS = \ + MemoryModule.$(OBJEXT) \ tclWin32Dll.$(OBJEXT) \ tclWinChan.$(OBJEXT) \ tclWinConsole.$(OBJEXT) \ @@ -702,6 +703,9 @@ tclZipfs.${OBJEXT}: $(GENERIC_DIR)/tclZipfs.c $(CC) -c $(CC_SWITCHES) -DBUILD_tcl \ $(ZLIB_INCLUDE) -I$(MINIZIP_DIR_NATIVE) @DEPARG@ $(CC_OBJNAME) +MemoryModule.${OBJEXT}: $(WIN_DIR)/MemoryModule.c + $(CC) -c $(CC_SWITCHES) -DUNICODE @DEPARG@ $(CC_OBJNAME) + # TIP #59, embedding of configuration information into the binary library. # diff --git a/win/MemoryModule.c b/win/MemoryModule.c new file mode 100644 index 00000000000..48245f2fd81 --- /dev/null +++ b/win/MemoryModule.c @@ -0,0 +1,758 @@ +/* + * Memory DLL loading code + * Version 0.0.4 + * + * Copyright (c) 2004-2014 by Joachim Bauch / mail@joachim-bauch.de + * http://www.joachim-bauch.de + * + * The contents of this file are subject to the Mozilla Public License Version + * 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is MemoryModule.c + * + * The Initial Developer of the Original Code is Joachim Bauch. + * + * Portions created by Joachim Bauch are Copyright (C) 2004-2014 + * Joachim Bauch. All Rights Reserved. + * + */ + +#ifndef __GNUC__ +// disable warnings about pointer <-> DWORD conversions +#pragma warning( disable : 4311 4312 ) +#endif + +#ifdef _WIN64 +#define POINTER_TYPE ULONGLONG +#else +#define POINTER_TYPE DWORD +#endif + +#include +#include +#include +#ifdef DEBUG_OUTPUT +#include +#endif + +#ifndef IMAGE_SIZEOF_BASE_RELOCATION +// Vista SDKs no longer define IMAGE_SIZEOF_BASE_RELOCATION!? +#define IMAGE_SIZEOF_BASE_RELOCATION (sizeof(IMAGE_BASE_RELOCATION)) +#endif + +#include "MemoryModule.h" + +typedef struct { + PIMAGE_NT_HEADERS headers; + unsigned char *codeBase; + HCUSTOMMODULE *modules; + int numModules; + int initialized; + CustomLoadLibraryFunc loadLibrary; + CustomGetProcAddressFunc getProcAddress; + CustomFreeLibraryFunc freeLibrary; + void *userdata; +} MEMORYMODULE, *PMEMORYMODULE; + +typedef BOOL (WINAPI *DllEntryProc)(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpReserved); + +#define GET_HEADER_DICTIONARY(module, idx) &(module)->headers->OptionalHeader.DataDirectory[idx] + +#ifdef DEBUG_OUTPUT +static void +OutputLastError(const char *msg) +{ + LPVOID tmp; + char *tmpmsg; + FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPTSTR)&tmp, 0, NULL); + tmpmsg = (char *)LocalAlloc(LPTR, strlen(msg) + strlen(tmp) + 3); + sprintf(tmpmsg, "%s: %s", msg, tmp); + OutputDebugString(tmpmsg); + LocalFree(tmpmsg); + LocalFree(tmp); +} +#endif + +static void +CopySections(const unsigned char *data, PIMAGE_NT_HEADERS old_headers, PMEMORYMODULE module) +{ + int i, size; + unsigned char *codeBase = module->codeBase; + unsigned char *dest; + PIMAGE_SECTION_HEADER section = IMAGE_FIRST_SECTION(module->headers); + for (i=0; iheaders->FileHeader.NumberOfSections; i++, section++) { + if (section->SizeOfRawData == 0) { + // section doesn't contain data in the dll itself, but may define + // uninitialized data + size = old_headers->OptionalHeader.SectionAlignment; + if (size > 0) { + dest = (unsigned char *)VirtualAlloc(codeBase + section->VirtualAddress, + size, + MEM_COMMIT, + PAGE_READWRITE); + + section->Misc.PhysicalAddress = (DWORD) (POINTER_TYPE) dest; + memset(dest, 0, size); + } + + // section is empty + continue; + } + + // commit memory block and copy data from dll + dest = (unsigned char *)VirtualAlloc(codeBase + section->VirtualAddress, + section->SizeOfRawData, + MEM_COMMIT, + PAGE_READWRITE); + memcpy(dest, data + section->PointerToRawData, section->SizeOfRawData); + section->Misc.PhysicalAddress = (DWORD) (POINTER_TYPE) dest; + } +} + +// Protection flags for memory pages (Executable, Readable, Writeable) +static int ProtectionFlags[2][2][2] = { + { + // not executable + {PAGE_NOACCESS, PAGE_WRITECOPY}, + {PAGE_READONLY, PAGE_READWRITE}, + }, { + // executable + {PAGE_EXECUTE, PAGE_EXECUTE_WRITECOPY}, + {PAGE_EXECUTE_READ, PAGE_EXECUTE_READWRITE}, + }, +}; + +static void +FinalizeSections(PMEMORYMODULE module) +{ + int i; + PIMAGE_SECTION_HEADER section = IMAGE_FIRST_SECTION(module->headers); +#ifdef _WIN64 + POINTER_TYPE imageOffset = (module->headers->OptionalHeader.ImageBase & 0xffffffff00000000); +#else + #define imageOffset 0 +#endif + + // loop through all sections and change access flags + for (i=0; iheaders->FileHeader.NumberOfSections; i++, section++) { + DWORD protect, oldProtect, size; + int executable = (section->Characteristics & IMAGE_SCN_MEM_EXECUTE) != 0; + int readable = (section->Characteristics & IMAGE_SCN_MEM_READ) != 0; + int writeable = (section->Characteristics & IMAGE_SCN_MEM_WRITE) != 0; + + if (section->Characteristics & IMAGE_SCN_MEM_DISCARDABLE) { + // section is not needed any more and can safely be freed + VirtualFree((LPVOID)((POINTER_TYPE)section->Misc.PhysicalAddress | imageOffset), section->SizeOfRawData, MEM_DECOMMIT); + continue; + } + + // determine protection flags based on characteristics + protect = ProtectionFlags[executable][readable][writeable]; + if (section->Characteristics & IMAGE_SCN_MEM_NOT_CACHED) { + protect |= PAGE_NOCACHE; + } + + // determine size of region + size = section->SizeOfRawData; + if (size == 0) { + if (section->Characteristics & IMAGE_SCN_CNT_INITIALIZED_DATA) { + size = module->headers->OptionalHeader.SizeOfInitializedData; + } else if (section->Characteristics & IMAGE_SCN_CNT_UNINITIALIZED_DATA) { + size = module->headers->OptionalHeader.SizeOfUninitializedData; + } + } + + if (size > 0) { + // change memory access flags + if (VirtualProtect((LPVOID)((POINTER_TYPE)section->Misc.PhysicalAddress | imageOffset), size, protect, &oldProtect) == 0) { +#ifdef DEBUG_OUTPUT + OutputLastError("Error protecting memory page") +#endif + }; + } + } +#ifndef _WIN64 +#undef imageOffset +#endif +} + +static void +ExecuteTLS(PMEMORYMODULE module) +{ + unsigned char *codeBase = module->codeBase; + + PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_TLS); + if (directory->VirtualAddress > 0) { + PIMAGE_TLS_DIRECTORY tls = (PIMAGE_TLS_DIRECTORY) (codeBase + directory->VirtualAddress); + PIMAGE_TLS_CALLBACK* callback = (PIMAGE_TLS_CALLBACK *) tls->AddressOfCallBacks; + if (callback) { + while (*callback) { + (*callback)((LPVOID) codeBase, DLL_PROCESS_ATTACH, NULL); + callback++; + } + } + } +} + +static void +PerformBaseRelocation(PMEMORYMODULE module, SIZE_T delta) +{ + DWORD i; + unsigned char *codeBase = module->codeBase; + + PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_BASERELOC); + if (directory->Size > 0) { + PIMAGE_BASE_RELOCATION relocation = (PIMAGE_BASE_RELOCATION) (codeBase + directory->VirtualAddress); + for (; relocation->VirtualAddress > 0; ) { + unsigned char *dest = codeBase + relocation->VirtualAddress; + unsigned short *relInfo = (unsigned short *)((unsigned char *)relocation + IMAGE_SIZEOF_BASE_RELOCATION); + for (i=0; i<((relocation->SizeOfBlock-IMAGE_SIZEOF_BASE_RELOCATION) / 2); i++, relInfo++) { + DWORD *patchAddrHL; +#ifdef _WIN64 + ULONGLONG *patchAddr64; +#endif + int type, offset; + + // the upper 4 bits define the type of relocation + type = *relInfo >> 12; + // the lower 12 bits define the offset + offset = *relInfo & 0xfff; + + switch (type) + { + case IMAGE_REL_BASED_ABSOLUTE: + // skip relocation + break; + + case IMAGE_REL_BASED_HIGHLOW: + // change complete 32 bit address + patchAddrHL = (DWORD *) (dest + offset); + *patchAddrHL += (DWORD) delta; + break; + +#ifdef _WIN64 + case IMAGE_REL_BASED_DIR64: + patchAddr64 = (ULONGLONG *) (dest + offset); + *patchAddr64 += (ULONGLONG) delta; + break; +#endif + + default: + //printf("Unknown relocation: %d\n", type); + break; + } + } + + // advance to next relocation block + relocation = (PIMAGE_BASE_RELOCATION) (((char *) relocation) + relocation->SizeOfBlock); + } + } +} + +static int +BuildImportTable(PMEMORYMODULE module) +{ + int result=1; + unsigned char *codeBase = module->codeBase; + HCUSTOMMODULE *tmp; + + PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_IMPORT); + if (directory->Size > 0) { + PIMAGE_IMPORT_DESCRIPTOR importDesc = (PIMAGE_IMPORT_DESCRIPTOR) (codeBase + directory->VirtualAddress); + for (; !IsBadReadPtr(importDesc, sizeof(IMAGE_IMPORT_DESCRIPTOR)) && importDesc->Name; importDesc++) { + POINTER_TYPE *thunkRef; + FARPROC *funcRef; + HCUSTOMMODULE handle = module->loadLibrary((LPCSTR) (codeBase + importDesc->Name), module->userdata); + if (handle == NULL) { + SetLastError(ERROR_MOD_NOT_FOUND); + result = 0; + break; + } + + tmp = (HCUSTOMMODULE *) realloc(module->modules, (module->numModules+1)*(sizeof(HCUSTOMMODULE))); + if (tmp == NULL) { + module->freeLibrary(handle, module->userdata); + SetLastError(ERROR_OUTOFMEMORY); + result = 0; + break; + } + module->modules = tmp; + + module->modules[module->numModules++] = handle; + if (importDesc->OriginalFirstThunk) { + thunkRef = (POINTER_TYPE *) (codeBase + importDesc->OriginalFirstThunk); + funcRef = (FARPROC *) (codeBase + importDesc->FirstThunk); + } else { + // no hint table + thunkRef = (POINTER_TYPE *) (codeBase + importDesc->FirstThunk); + funcRef = (FARPROC *) (codeBase + importDesc->FirstThunk); + } + for (; *thunkRef; thunkRef++, funcRef++) { + if (IMAGE_SNAP_BY_ORDINAL(*thunkRef)) { + *funcRef = module->getProcAddress(handle, (LPCSTR)IMAGE_ORDINAL(*thunkRef), module->userdata); + } else { + PIMAGE_IMPORT_BY_NAME thunkData = (PIMAGE_IMPORT_BY_NAME) (codeBase + (*thunkRef)); + *funcRef = module->getProcAddress(handle, (LPCSTR)&thunkData->Name, module->userdata); + } + if (*funcRef == 0) { + result = 0; + break; + } + } + + if (!result) { + module->freeLibrary(handle, module->userdata); + SetLastError(ERROR_PROC_NOT_FOUND); + break; + } + } + } + + return result; +} + +static HCUSTOMMODULE _LoadLibrary(LPCSTR filename, void *userdata) +{ + HMODULE result = LoadLibraryA(filename); + (void)userdata; + if (result == NULL) { + return NULL; + } + + return (HCUSTOMMODULE) result; +} + +static FARPROC _GetProcAddress(HCUSTOMMODULE module, LPCSTR name, void *userdata) +{ + (void)userdata; + return (FARPROC) GetProcAddress((HMODULE) module, name); +} + +static void _FreeLibrary(HCUSTOMMODULE module, void *userdata) +{ + (void)userdata; + FreeLibrary((HMODULE) module); +} + +HMEMORYMODULE MemoryLoadLibrary(const void *data) +{ + return MemoryLoadLibraryEx(data, _LoadLibrary, _GetProcAddress, _FreeLibrary, NULL); +} + +HMEMORYMODULE MemoryLoadLibraryEx(const void *data, + CustomLoadLibraryFunc loadLibrary, + CustomGetProcAddressFunc getProcAddress, + CustomFreeLibraryFunc freeLibrary, + void *userdata) +{ + PMEMORYMODULE result; + PIMAGE_DOS_HEADER dos_header; + PIMAGE_NT_HEADERS old_header; + unsigned char *code, *headers; + SIZE_T locationDelta; + DllEntryProc DllEntry; + BOOL successfull; + + dos_header = (PIMAGE_DOS_HEADER)data; + if (dos_header->e_magic != IMAGE_DOS_SIGNATURE) { + SetLastError(ERROR_BAD_EXE_FORMAT); + return NULL; + } + + old_header = (PIMAGE_NT_HEADERS)&((const unsigned char *)(data))[dos_header->e_lfanew]; + if (old_header->Signature != IMAGE_NT_SIGNATURE) { + SetLastError(ERROR_BAD_EXE_FORMAT); + return NULL; + } + + // reserve memory for image of library + // XXX: is it correct to commit the complete memory region at once? + // calling DllEntry raises an exception if we don't... + code = (unsigned char *)VirtualAlloc((LPVOID)(old_header->OptionalHeader.ImageBase), + old_header->OptionalHeader.SizeOfImage, + MEM_RESERVE | MEM_COMMIT, + PAGE_READWRITE); + + if (code == NULL) { + // try to allocate memory at arbitrary position + code = (unsigned char *)VirtualAlloc(NULL, + old_header->OptionalHeader.SizeOfImage, + MEM_RESERVE | MEM_COMMIT, + PAGE_READWRITE); + if (code == NULL) { + SetLastError(ERROR_OUTOFMEMORY); + return NULL; + } + } + + result = (PMEMORYMODULE)HeapAlloc(GetProcessHeap(), 0, sizeof(MEMORYMODULE)); + if (result == NULL) { + SetLastError(ERROR_OUTOFMEMORY); + VirtualFree(code, 0, MEM_RELEASE); + return NULL; + } + + result->codeBase = code; + result->numModules = 0; + result->modules = NULL; + result->initialized = 0; + result->loadLibrary = loadLibrary; + result->getProcAddress = getProcAddress; + result->freeLibrary = freeLibrary; + result->userdata = userdata; + + // commit memory for headers + headers = (unsigned char *)VirtualAlloc(code, + old_header->OptionalHeader.SizeOfHeaders, + MEM_COMMIT, + PAGE_READWRITE); + + // copy PE header to code + memcpy(headers, dos_header, old_header->OptionalHeader.SizeOfHeaders); + result->headers = (PIMAGE_NT_HEADERS)&((const unsigned char *)(headers))[dos_header->e_lfanew]; + + // update position + result->headers->OptionalHeader.ImageBase = (POINTER_TYPE)code; + + // copy sections from DLL file block to new memory location + CopySections((const unsigned char *) data, old_header, result); + + // adjust base address of imported data + locationDelta = (SIZE_T)(code - old_header->OptionalHeader.ImageBase); + if (locationDelta != 0) { + PerformBaseRelocation(result, locationDelta); + } + + // load required dlls and adjust function table of imports + if (!BuildImportTable(result)) { + goto error; + } + + // mark memory pages depending on section headers and release + // sections that are marked as "discardable" + FinalizeSections(result); + + // TLS callbacks are executed BEFORE the main loading + ExecuteTLS(result); + + // get entry point of loaded library + if (result->headers->OptionalHeader.AddressOfEntryPoint != 0) { + DllEntry = (DllEntryProc) (code + result->headers->OptionalHeader.AddressOfEntryPoint); + // notify library about attaching to process + successfull = (*DllEntry)((HINSTANCE)code, DLL_PROCESS_ATTACH, 0); + if (!successfull) { + SetLastError(ERROR_DLL_INIT_FAILED); + goto error; + } + result->initialized = 1; + } + + return (HMEMORYMODULE)result; + +error: + // cleanup + MemoryFreeLibrary(result); + return NULL; +} + +FARPROC MemoryGetProcAddress(HMEMORYMODULE module, LPCSTR name) +{ + unsigned char *codeBase = ((PMEMORYMODULE)module)->codeBase; + int idx=-1; + DWORD i, *nameRef; + WORD *ordinal; + PIMAGE_EXPORT_DIRECTORY exports; + PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY((PMEMORYMODULE)module, IMAGE_DIRECTORY_ENTRY_EXPORT); + if (directory->Size == 0) { + // no export table found + SetLastError(ERROR_PROC_NOT_FOUND); + return NULL; + } + + exports = (PIMAGE_EXPORT_DIRECTORY) (codeBase + directory->VirtualAddress); + if (exports->NumberOfNames == 0 || exports->NumberOfFunctions == 0) { + // DLL doesn't export anything + SetLastError(ERROR_PROC_NOT_FOUND); + return NULL; + } + + // search function name in list of exported names + nameRef = (DWORD *) (codeBase + exports->AddressOfNames); + ordinal = (WORD *) (codeBase + exports->AddressOfNameOrdinals); + for (i=0; iNumberOfNames; i++, nameRef++, ordinal++) { + if (_stricmp(name, (const char *) (codeBase + (*nameRef))) == 0) { + idx = *ordinal; + break; + } + } + + if (idx == -1) { + // exported symbol not found + SetLastError(ERROR_PROC_NOT_FOUND); + return NULL; + } + + if ((DWORD)idx > exports->NumberOfFunctions) { + // name <-> ordinal number don't match + SetLastError(ERROR_PROC_NOT_FOUND); + return NULL; + } + + // AddressOfFunctions contains the RVAs to the "real" functions + return (FARPROC) (codeBase + (*(DWORD *) (codeBase + exports->AddressOfFunctions + (idx*4)))); +} + +void MemoryFreeLibrary(HMEMORYMODULE mod) +{ + int i; + PMEMORYMODULE module = (PMEMORYMODULE)mod; + + if (module != NULL) { + if (module->initialized != 0) { + // notify library about detaching from process + DllEntryProc DllEntry = (DllEntryProc) (module->codeBase + module->headers->OptionalHeader.AddressOfEntryPoint); + (*DllEntry)((HINSTANCE)module->codeBase, DLL_PROCESS_DETACH, 0); + module->initialized = 0; + } + + if (module->modules != NULL) { + // free previously opened libraries + for (i=0; inumModules; i++) { + if (module->modules[i] != NULL) { + module->freeLibrary(module->modules[i], module->userdata); + } + } + + free(module->modules); + } + + if (module->codeBase != NULL) { + // release memory of library + VirtualFree(module->codeBase, 0, MEM_RELEASE); + } + + HeapFree(GetProcessHeap(), 0, module); + } +} + +#define DEFAULT_LANGUAGE MAKELANGID(LANG_NEUTRAL, SUBLANG_NEUTRAL) + +HMEMORYRSRC MemoryFindResource(HMEMORYMODULE module, LPCTSTR name, LPCTSTR type) +{ + return MemoryFindResourceEx(module, name, type, DEFAULT_LANGUAGE); +} + +static PIMAGE_RESOURCE_DIRECTORY_ENTRY _MemorySearchResourceEntry( + void *root, + PIMAGE_RESOURCE_DIRECTORY resources, + LPCTSTR key) +{ + PIMAGE_RESOURCE_DIRECTORY_ENTRY entries = (PIMAGE_RESOURCE_DIRECTORY_ENTRY) (resources + 1); + PIMAGE_RESOURCE_DIRECTORY_ENTRY result = NULL; + DWORD start; + DWORD end; + DWORD middle; + + if (!IS_INTRESOURCE(key) && key[0] == TEXT('#')) { + // special case: resource id given as string + TCHAR *endpos = NULL; +#if defined(UNICODE) + long int tmpkey = (WORD) wcstol((TCHAR *) &key[1], &endpos, 10); +#else + long int tmpkey = (WORD) strtol((TCHAR *) &key[1], &endpos, 10); +#endif + if (tmpkey <= 0xffff && lstrlen(endpos) == 0) { + key = MAKEINTRESOURCE(tmpkey); + } + } + + // entries are stored as ordered list of named entries, + // followed by an ordered list of id entries - we can do + // a binary search to find faster... + if (IS_INTRESOURCE(key)) { + WORD check = (WORD) (POINTER_TYPE) key; + start = resources->NumberOfNamedEntries; + end = start + resources->NumberOfIdEntries; + + while (end > start) { + WORD entryName; + middle = (start + end) >> 1; + entryName = (WORD) entries[middle].Name; + if (check < entryName) { + end = (end != middle ? middle : middle-1); + } else if (check > entryName) { + start = (start != middle ? middle : middle+1); + } else { + result = &entries[middle]; + break; + } + } + } else { +#if !defined(UNICODE) + char *searchKey = NULL; + int searchKeyLength = 0; +#endif + start = 0; + end = resources->NumberOfIdEntries; + while (end > start) { + // resource names are always stored using 16bit characters + int cmp; + PIMAGE_RESOURCE_DIR_STRING_U resourceString; + middle = (start + end) >> 1; + resourceString = (PIMAGE_RESOURCE_DIR_STRING_U) (((char *) root) + (entries[middle].Name & 0x7FFFFFFF)); +#if !defined(UNICODE) + if (searchKey == NULL || searchKeyLength < resourceString->Length) { + void *tmp = realloc(searchKey, resourceString->Length); + if (tmp == NULL) { + break; + } + + searchKey = (char *) tmp; + } + wcstombs(searchKey, resourceString->NameString, resourceString->Length); + cmp = strncmp(key, searchKey, resourceString->Length); +#else + cmp = wcsncmp(key, resourceString->NameString, resourceString->Length); +#endif + if (cmp < 0) { + end = (middle != end ? middle : middle-1); + } else if (cmp > 0) { + start = (middle != start ? middle : middle+1); + } else { + result = &entries[middle]; + break; + } + } +#if !defined(UNICODE) + free(searchKey); +#endif + } + + + return result; +} + +HMEMORYRSRC MemoryFindResourceEx(HMEMORYMODULE module, LPCTSTR name, LPCTSTR type, WORD language) +{ + unsigned char *codeBase = ((PMEMORYMODULE) module)->codeBase; + PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY((PMEMORYMODULE) module, IMAGE_DIRECTORY_ENTRY_RESOURCE); + PIMAGE_RESOURCE_DIRECTORY rootResources; + PIMAGE_RESOURCE_DIRECTORY nameResources; + PIMAGE_RESOURCE_DIRECTORY typeResources; + PIMAGE_RESOURCE_DIRECTORY_ENTRY foundType; + PIMAGE_RESOURCE_DIRECTORY_ENTRY foundName; + PIMAGE_RESOURCE_DIRECTORY_ENTRY foundLanguage; + if (directory->Size == 0) { + // no resource table found + SetLastError(ERROR_RESOURCE_DATA_NOT_FOUND); + return NULL; + } + + if (language == DEFAULT_LANGUAGE) { + // use language from current thread + language = LANGIDFROMLCID(GetThreadLocale()); + } + + // resources are stored as three-level tree + // - first node is the type + // - second node is the name + // - third node is the language + rootResources = (PIMAGE_RESOURCE_DIRECTORY) (codeBase + directory->VirtualAddress); + foundType = _MemorySearchResourceEntry(rootResources, rootResources, type); + if (foundType == NULL) { + SetLastError(ERROR_RESOURCE_TYPE_NOT_FOUND); + return NULL; + } + + typeResources = (PIMAGE_RESOURCE_DIRECTORY) (codeBase + directory->VirtualAddress + (foundType->OffsetToData & 0x7fffffff)); + foundName = _MemorySearchResourceEntry(rootResources, typeResources, name); + if (foundName == NULL) { + SetLastError(ERROR_RESOURCE_NAME_NOT_FOUND); + return NULL; + } + + nameResources = (PIMAGE_RESOURCE_DIRECTORY) (codeBase + directory->VirtualAddress + (foundName->OffsetToData & 0x7fffffff)); + foundLanguage = _MemorySearchResourceEntry(rootResources, nameResources, (LPCTSTR) (POINTER_TYPE) language); + if (foundLanguage == NULL) { + // requested language not found, use first available + if (nameResources->NumberOfIdEntries == 0) { + SetLastError(ERROR_RESOURCE_LANG_NOT_FOUND); + return NULL; + } + + foundLanguage = (PIMAGE_RESOURCE_DIRECTORY_ENTRY) (nameResources + 1); + } + + return (codeBase + directory->VirtualAddress + (foundLanguage->OffsetToData & 0x7fffffff)); +} + +DWORD MemorySizeofResource(HMEMORYMODULE module, HMEMORYRSRC resource) +{ + PIMAGE_RESOURCE_DATA_ENTRY entry = (PIMAGE_RESOURCE_DATA_ENTRY) resource; + (void)module; + + return entry->Size; +} + +LPVOID MemoryLoadResource(HMEMORYMODULE module, HMEMORYRSRC resource) +{ + unsigned char *codeBase = ((PMEMORYMODULE) module)->codeBase; + PIMAGE_RESOURCE_DATA_ENTRY entry = (PIMAGE_RESOURCE_DATA_ENTRY) resource; + + return codeBase + entry->OffsetToData; +} + +int +MemoryLoadString(HMEMORYMODULE module, UINT id, LPTSTR buffer, int maxsize) +{ + return MemoryLoadStringEx(module, id, buffer, maxsize, DEFAULT_LANGUAGE); +} + +int +MemoryLoadStringEx(HMEMORYMODULE module, UINT id, LPTSTR buffer, int maxsize, WORD language) +{ + HMEMORYRSRC resource; + PIMAGE_RESOURCE_DIR_STRING_U data; + DWORD size; + if (maxsize == 0) { + return 0; + } + + resource = MemoryFindResourceEx(module, MAKEINTRESOURCE((id >> 4) + 1), RT_STRING, language); + if (resource == NULL) { + buffer[0] = 0; + return 0; + } + + data = (PIMAGE_RESOURCE_DIR_STRING_U) MemoryLoadResource(module, resource); + id = id & 0x0f; + while (id--) { + data = (PIMAGE_RESOURCE_DIR_STRING_U) (((char *) data) + (data->Length + 1) * sizeof(WCHAR)); + } + if (data->Length == 0) { + SetLastError(ERROR_RESOURCE_NAME_NOT_FOUND); + buffer[0] = 0; + return 0; + } + + size = data->Length; + if (size >= (DWORD) maxsize) { + size = maxsize; + } else { + buffer[size] = 0; + } +#if defined(UNICODE) + wcsncpy(buffer, data->NameString, size); +#else + wcstombs(buffer, data->NameString, size); +#endif + return size; +} diff --git a/win/MemoryModule.h b/win/MemoryModule.h new file mode 100644 index 00000000000..adeacb6521c --- /dev/null +++ b/win/MemoryModule.h @@ -0,0 +1,109 @@ +/* + * Memory DLL loading code + * Version 0.0.4 + * + * Copyright (c) 2004-2014 by Joachim Bauch / mail@joachim-bauch.de + * http://www.joachim-bauch.de + * + * The contents of this file are subject to the Mozilla Public License Version + * 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is MemoryModule.h + * + * The Initial Developer of the Original Code is Joachim Bauch. + * + * Portions created by Joachim Bauch are Copyright (C) 2004-2014 + * Joachim Bauch. All Rights Reserved. + * + */ + +#ifndef __MEMORY_MODULE_HEADER +#define __MEMORY_MODULE_HEADER + +#include + +typedef void *HMEMORYMODULE; + +typedef void *HMEMORYRSRC; + +typedef void *HCUSTOMMODULE; + +#ifdef __cplusplus +extern "C" { +#endif + +typedef HCUSTOMMODULE (*CustomLoadLibraryFunc)(LPCSTR, void *); +typedef FARPROC (*CustomGetProcAddressFunc)(HCUSTOMMODULE, LPCSTR, void *); +typedef void (*CustomFreeLibraryFunc)(HCUSTOMMODULE, void *); + +/** + * Load DLL from memory location. + * + * All dependencies are resolved using default LoadLibrary/GetProcAddress + * calls through the Windows API. + */ +HMEMORYMODULE MemoryLoadLibrary(const void *); + +/** + * Load DLL from memory location using custom dependency resolvers. + * + * Dependencies will be resolved using passed callback methods. + */ +HMEMORYMODULE MemoryLoadLibraryEx(const void *, + CustomLoadLibraryFunc, + CustomGetProcAddressFunc, + CustomFreeLibraryFunc, + void *); + +/** + * Get address of exported method. + */ +FARPROC MemoryGetProcAddress(HMEMORYMODULE, LPCSTR); + +/** + * Free previously loaded DLL. + */ +void MemoryFreeLibrary(HMEMORYMODULE); + +/** + * Find the location of a resource with the specified type and name. + */ +HMEMORYRSRC MemoryFindResource(HMEMORYMODULE, LPCTSTR, LPCTSTR); + +/** + * Find the location of a resource with the specified type, name and language. + */ +HMEMORYRSRC MemoryFindResourceEx(HMEMORYMODULE, LPCTSTR, LPCTSTR, WORD); + +/** + * Get the size of the resource in bytes. + */ +DWORD MemorySizeofResource(HMEMORYMODULE, HMEMORYRSRC); + +/** + * Get a pointer to the contents of the resource. + */ +LPVOID MemoryLoadResource(HMEMORYMODULE, HMEMORYRSRC); + +/** + * Load a string resource. + */ +int MemoryLoadString(HMEMORYMODULE, UINT, LPTSTR, int); + +/** + * Load a string resource with a given language. + */ +int MemoryLoadStringEx(HMEMORYMODULE, UINT, LPTSTR, int, WORD); + +#ifdef __cplusplus +} +#endif + +#endif // __MEMORY_MODULE_HEADER diff --git a/win/makefile.vc b/win/makefile.vc index 951ff39961e..aa48b14eea4 100644 --- a/win/makefile.vc +++ b/win/makefile.vc @@ -429,6 +429,7 @@ TOMMATHOBJS = $(OUT_DIR)\tommath.lib !endif PLATFORMOBJS = \ + $(TMP_DIR)\MemoryModule.obj \ $(TMP_DIR)\tclWin32Dll.obj \ $(TMP_DIR)\tclWinChan.obj \ $(TMP_DIR)\tclWinConsole.obj \ @@ -472,7 +473,7 @@ LIBTCLVFS = $(OUT_DIR)\$(LIBTCLVFSSUBDIR) # Additional include and C macro definitions for the implicit rules # defined in rules.vc PRJ_INCLUDES = -I"$(TOMMATHDIR)" -PRJ_DEFINES = /DMP_PREC=4 /Dinline=__inline /D_CRT_SECURE_NO_DEPRECATE /D_CRT_NONSTDC_NO_DEPRECATE /DMP_FIXED_CUTOFFS +PRJ_DEFINES = /DMP_PREC=4 /Dinline=__inline /D_CRT_SECURE_NO_DEPRECATE /D_CRT_NONSTDC_NO_DEPRECATE /DMP_FIXED_CUTOFFS /DTCL_LOAD_FROM_MEMORY !if $(STATIC_BUILD) PRJ_DEFINES = $(PRJ_DEFINES) /DTCL_WITH_INTERNAL_ZLIB @@ -914,6 +915,9 @@ $(TMP_DIR)\tclZipfs.obj: $(GENERICDIR)\tclZipfs.c $(TMP_DIR)\tclZlib.obj: $(GENERICDIR)\tclZlib.c $(cc32) $(pkgcflags) -I$(COMPATDIR)\zlib -Fo$@ $? +$(TMP_DIR)\MemoryModule.obj: $(WIN_DIR)\MemoryModule.c + $(cc32) $(pkgcflags) /DUNICODE -Fo$@ $? + # Following the lead of the autoconf based make, we define the # CFG_RUNTIME_*DIR flags specifically for tclPkgConfig # and not as part of the global defines. These are all defined diff --git a/win/tclWinLoad.c b/win/tclWinLoad.c index b0c4c3b4f60..4874f7319f6 100644 --- a/win/tclWinLoad.c +++ b/win/tclWinLoad.c @@ -12,6 +12,8 @@ */ #include "tclWinInt.h" +#define UNICODE +#include "MemoryModule.h" /* * Native name of the directory in the native filesystem where DLLs used in @@ -395,6 +397,74 @@ InitDLLDirectoryName(void) return TCL_OK; } +#ifdef TCL_LOAD_FROM_MEMORY + +MODULE_SCOPE void * +TclpLoadMemoryGetBuffer( + TCL_UNUSED(Tcl_Interp *), + size_t size) +{ + return Tcl_Alloc(size); +} + +static void +UnloadMemory( + Tcl_LoadHandle loadHandle) /* loadHandle returned by a previous call to + * TclpDlopen(). The loadHandle is a token + * that represents the loaded file. */ +{ + struct Tcl_LoadHandle_ *handlePtr = (struct Tcl_LoadHandle_ *)loadHandle; + + MemoryFreeLibrary(loadHandle->clientData); + Tcl_Free(handlePtr); +} + +void* FindMemSymbol(Tcl_Interp* interp, Tcl_LoadHandle loadHandle, + const char* symbol) +{ + struct Tcl_LoadHandle_ *handlePtr = (struct Tcl_LoadHandle_ *)loadHandle; + + void *res = MemoryGetProcAddress(handlePtr->clientData, symbol); + + if (!res && interp) { + Tcl_SetObjResult(interp, Tcl_ObjPrintf( + "cannot find symbol \"%s\" in memory-loaded dll", symbol)); + Tcl_SetErrorCode(interp, "TCL", "LOOKUP", "LOAD_SYMBOL", symbol, + (char *)NULL); + } + return res; +} + +MODULE_SCOPE int +TclpLoadMemory( + Tcl_Interp *interp, + void *data, + TCL_UNUSED(int), + TCL_UNUSED(int), + Tcl_LoadHandle *loadHandle, + Tcl_FSUnloadFileProc **unloadProcPtr, + TCL_UNUSED(int)) +{ + struct Tcl_LoadHandle_ *handlePtr; + void *hInstance; + + hInstance = MemoryLoadLibrary(data); + if (!hInstance) { + Tcl_SetObjResult(interp, Tcl_NewStringObj( + "cannot load dll in memory", -1)); + Tcl_SetErrorCode(interp, "WIN_LOAD", "LOAD_MEMORY", (char *)NULL); + } + handlePtr = (struct Tcl_LoadHandle_ *)Tcl_Alloc(sizeof(struct Tcl_LoadHandle_)); + handlePtr->clientData = hInstance; + handlePtr->findSymbolProcPtr = &FindMemSymbol; + handlePtr->unloadFileProcPtr = &UnloadMemory; + *loadHandle = handlePtr; + *unloadProcPtr = &UnloadMemory; + return TCL_OK; +} + +#endif /* TCL_LOAD_FROM_MEMORY */ + /* * Local Variables: * mode: c From b70529008e1ba976e24f98dfe37b722207d7d97c Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Mon, 18 Nov 2024 15:35:39 +0000 Subject: [PATCH 03/33] Code cleanup --- win/tclWinLoad.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/win/tclWinLoad.c b/win/tclWinLoad.c index 4874f7319f6..7d8fb378fd0 100644 --- a/win/tclWinLoad.c +++ b/win/tclWinLoad.c @@ -413,18 +413,14 @@ UnloadMemory( * TclpDlopen(). The loadHandle is a token * that represents the loaded file. */ { - struct Tcl_LoadHandle_ *handlePtr = (struct Tcl_LoadHandle_ *)loadHandle; - MemoryFreeLibrary(loadHandle->clientData); - Tcl_Free(handlePtr); + Tcl_Free(loadHandle); } void* FindMemSymbol(Tcl_Interp* interp, Tcl_LoadHandle loadHandle, const char* symbol) { - struct Tcl_LoadHandle_ *handlePtr = (struct Tcl_LoadHandle_ *)loadHandle; - - void *res = MemoryGetProcAddress(handlePtr->clientData, symbol); + void *res = MemoryGetProcAddress(loadHandle->clientData, symbol); if (!res && interp) { Tcl_SetObjResult(interp, Tcl_ObjPrintf( From 2189aafac635bdfdd6d5d86f56274f51f0859417 Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Mon, 18 Nov 2024 20:00:58 +0000 Subject: [PATCH 04/33] Missing "return TCL_ERROR" --- win/tclWinLoad.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/win/tclWinLoad.c b/win/tclWinLoad.c index 7d8fb378fd0..7566fa55e22 100644 --- a/win/tclWinLoad.c +++ b/win/tclWinLoad.c @@ -417,8 +417,8 @@ UnloadMemory( Tcl_Free(loadHandle); } -void* FindMemSymbol(Tcl_Interp* interp, Tcl_LoadHandle loadHandle, - const char* symbol) +void *FindMemSymbol(Tcl_Interp* interp, Tcl_LoadHandle loadHandle, + const char *symbol) { void *res = MemoryGetProcAddress(loadHandle->clientData, symbol); @@ -441,7 +441,7 @@ TclpLoadMemory( Tcl_FSUnloadFileProc **unloadProcPtr, TCL_UNUSED(int)) { - struct Tcl_LoadHandle_ *handlePtr; + Tcl_LoadHandle handlePtr; void *hInstance; hInstance = MemoryLoadLibrary(data); @@ -449,8 +449,9 @@ TclpLoadMemory( Tcl_SetObjResult(interp, Tcl_NewStringObj( "cannot load dll in memory", -1)); Tcl_SetErrorCode(interp, "WIN_LOAD", "LOAD_MEMORY", (char *)NULL); + return TCL_ERROR; } - handlePtr = (struct Tcl_LoadHandle_ *)Tcl_Alloc(sizeof(struct Tcl_LoadHandle_)); + handlePtr = (Tcl_LoadHandle)Tcl_Alloc(sizeof(struct Tcl_LoadHandle_)); handlePtr->clientData = hInstance; handlePtr->findSymbolProcPtr = &FindMemSymbol; handlePtr->unloadFileProcPtr = &UnloadMemory; From 0b8dae75ca703a76edd70417384a432edb2222f7 Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Thu, 21 Nov 2024 15:32:51 +0000 Subject: [PATCH 05/33] backout [bc7fbd8e2e]. Testing done. --- win/Makefile.in | 9 ++------- win/makefile.vc | 9 --------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/win/Makefile.in b/win/Makefile.in index 8654904003d..932a82101a4 100644 --- a/win/Makefile.in +++ b/win/Makefile.in @@ -550,13 +550,8 @@ ${TCL_ZIP_FILE}: ${ZIP_INSTALL_OBJS} ${DDE_DLL_FILE} ${REG_DLL_FILE} @( \ $(COPY) -a $(TOP_DIR)/library/* ${TCL_VFS_PATH}; \ $(COPY) -a ${TCL_VFS_PATH}/manifest.txt ${TCL_VFS_PATH}/pkgIndex.tcl; \ - if test "${ZIPFS_BUILD}" != "2" ; then \ - $(COPY) ${DDE_DLL_FILE} ${TCL_VFS_PATH}/dde; \ - $(COPY) ${REG_DLL_FILE} ${TCL_VFS_PATH}/registry; \ - else \ - rm -rf ${TCL_VFS_PATH}/dde; \ - rm -rf ${TCL_VFS_PATH}/registry; \ - fi \ + rm -rf ${TCL_VFS_PATH}/dde; \ + rm -rf ${TCL_VFS_PATH}/registry; \ ) (zip=`(realpath '${NATIVE_ZIP}' || readlink -m '${NATIVE_ZIP}') 2>/dev/null || \ (echo '${NATIVE_ZIP}' | sed "s?^\./?$$(pwd)/?")`; \ diff --git a/win/makefile.vc b/win/makefile.vc index aa48b14eea4..f48aa853275 100644 --- a/win/makefile.vc +++ b/win/makefile.vc @@ -662,16 +662,11 @@ $(TCLSCRIPTZIP): $(TCLLIB) $(TCLSH) dlls @$(MKDIR) "$(LIBTCLVFS)" @$(CPYDIR) $(LIBDIR) "$(LIBTCLVFS)\tcl_library" @move /y "$(LIBTCLVFS)\tcl_library\manifest.txt" "$(LIBTCLVFS)\tcl_library\pkgIndex.tcl" > NUL -!if $(STATIC_BUILD) # Remove the registry and dde directories as the DLLS are still external @del "$(LIBTCLVFS)\tcl_library\registry\pkgIndex.tcl" @rmdir "$(LIBTCLVFS)\tcl_library\registry" @del "$(LIBTCLVFS)\tcl_library\dde\pkgIndex.tcl" @rmdir "$(LIBTCLVFS)\tcl_library\dde" -!else - @$(COPY) $(TCLDDELIB) "$(LIBTCLVFS)\tcl_library\dde - @$(COPY) $(TCLREGLIB) "$(LIBTCLVFS)\tcl_library\registry -!endif @echo cd {$(OUT_DIR)} > "$(OUT_DIR)\zipper.tcl" @echo file delete -force {$(@F)} >> "$(OUT_DIR)\zipper.tcl" @echo zipfs mkzip {$(@F)} {$(LIBTCLVFSSUBDIR)} {$(LIBTCLVFSSUBDIR)} >> "$(OUT_DIR)\zipper.tcl" @@ -1133,17 +1128,13 @@ install-libraries: tclConfig tcl-nmake install-msgs install-tzdata "$(MODULE_INSTALL_DIR)\9.0\platform\shell-$(PKG_SHELL_VER).tm" !endif @echo Installing $(TCLDDELIBNAME) -!if !$(STATIC_BUILD) @$(CPY) "$(TCLDDELIB)" "$(LIB_INSTALL_DIR)\dde$(DDEDOTVERSION)\" @$(CPY) "$(ROOT)\library\dde\pkgIndex.tcl" \ "$(LIB_INSTALL_DIR)\dde$(DDEDOTVERSION)\" -!endif @echo Installing $(TCLREGLIBNAME) -!if !$(STATIC_BUILD) @$(CPY) "$(TCLREGLIB)" "$(LIB_INSTALL_DIR)\registry$(REGDOTVERSION)\" @$(CPY) "$(ROOT)\library\registry\pkgIndex.tcl" \ "$(LIB_INSTALL_DIR)\registry$(REGDOTVERSION)\" -!endif !if !$(TCL_EMBED_SCRIPTS) @echo Installing encodings @$(CPY) "$(ROOT)\library\encoding\*.enc" \ From 75485d3ff4570c1f2327119e2660847ea6df11e2 Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Fri, 22 Nov 2024 14:29:39 +0000 Subject: [PATCH 06/33] Update changes.md --- changes.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/changes.md b/changes.md index a7ebf3099f3..7fa5a48d91b 100644 --- a/changes.md +++ b/changes.md @@ -16,6 +16,7 @@ Release Tcl 9.0.1 arises from the check-in with tag `core-9-0-1`. - [cannot build .chm help file (Windows)](https://core.tcl-lang.org/tcl/tktview/bb110c) - [TIP 701 - Tcl_FSTildeExpand C API](https://core.tcl-lang.org/tips/doc/trunk/tip/701.md) - [buffer overwrite for non-BMP characters in utf-16](https://core.tcl-lang.org/tcl/tktview/66da4d) + - [load library (dll) from zipfs-library causes a leak in temporary folder](https://core.tcl-lang.org/tcl/tktview/a8e4f7) Release Tcl 9.0.0 arises from the check-in with tag `core-9-0-0`. @@ -154,5 +155,3 @@ writing Tcl scripts. # Known bugs - [changed behaviour wrt command names, namespaces and resolution](https://core.tcl-lang.org/tcl/tktview/f14b33) - [windows dos device paths inconsistencies and missing functionality](https://core.tcl-lang.org/tcl/tktview/d8f121) - - [Temporary folder with file "tcl9registry13.dll" remains after "exit"](https://core.tcl-lang.org/tcl/tktview/6ce3c0) - From ec043f6b406d9c70f1ffe832f0932faf3504c50e Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Sat, 23 Nov 2024 16:20:38 +0000 Subject: [PATCH 07/33] Update MemoryModule to latest (master) version. --- win/Makefile.in | 2 +- win/MemoryModule.c | 990 ++++++++++++++++++++++++++++++++------------- win/MemoryModule.h | 75 +++- win/makefile.vc | 2 +- win/tclWinLoad.c | 4 +- 5 files changed, 788 insertions(+), 285 deletions(-) diff --git a/win/Makefile.in b/win/Makefile.in index 932a82101a4..be9d79565d2 100644 --- a/win/Makefile.in +++ b/win/Makefile.in @@ -699,7 +699,7 @@ tclZipfs.${OBJEXT}: $(GENERIC_DIR)/tclZipfs.c $(ZLIB_INCLUDE) -I$(MINIZIP_DIR_NATIVE) @DEPARG@ $(CC_OBJNAME) MemoryModule.${OBJEXT}: $(WIN_DIR)/MemoryModule.c - $(CC) -c $(CC_SWITCHES) -DUNICODE @DEPARG@ $(CC_OBJNAME) + $(CC) -c $(CC_SWITCHES) -DUNICODE -D_UNICODE @DEPARG@ $(CC_OBJNAME) # TIP #59, embedding of configuration information into the binary library. diff --git a/win/MemoryModule.c b/win/MemoryModule.c index 48245f2fd81..f6b2fe7ff7b 100644 --- a/win/MemoryModule.c +++ b/win/MemoryModule.c @@ -2,7 +2,7 @@ * Memory DLL loading code * Version 0.0.4 * - * Copyright (c) 2004-2014 by Joachim Bauch / mail@joachim-bauch.de + * Copyright (c) 2004-2015 by Joachim Bauch / mail@joachim-bauch.de * http://www.joachim-bauch.de * * The contents of this file are subject to the Mozilla Public License Version @@ -19,56 +19,122 @@ * * The Initial Developer of the Original Code is Joachim Bauch. * - * Portions created by Joachim Bauch are Copyright (C) 2004-2014 + * Portions created by Joachim Bauch are Copyright (C) 2004-2015 * Joachim Bauch. All Rights Reserved. * + * + * THeller: Added binary search in MemoryGetProcAddress function + * (#define USE_BINARY_SEARCH to enable it). This gives a very large + * speedup for libraries that exports lots of functions. + * + * These portions are Copyright (C) 2013 Thomas Heller. */ -#ifndef __GNUC__ -// disable warnings about pointer <-> DWORD conversions -#pragma warning( disable : 4311 4312 ) -#endif - -#ifdef _WIN64 -#define POINTER_TYPE ULONGLONG -#else -#define POINTER_TYPE DWORD -#endif - #include #include +#include #include #ifdef DEBUG_OUTPUT #include #endif +#ifdef _MSC_VER +// Disable warning about data -> function pointer conversion +#pragma warning(disable:4055) + // C4244: conversion from 'uintptr_t' to 'DWORD', possible loss of data. +#pragma warning(error: 4244) +// C4267: conversion from 'size_t' to 'int', possible loss of data. +#pragma warning(error: 4267) + +#define inline __inline +#endif + #ifndef IMAGE_SIZEOF_BASE_RELOCATION // Vista SDKs no longer define IMAGE_SIZEOF_BASE_RELOCATION!? #define IMAGE_SIZEOF_BASE_RELOCATION (sizeof(IMAGE_BASE_RELOCATION)) #endif +#ifdef _WIN64 +#define HOST_MACHINE IMAGE_FILE_MACHINE_AMD64 +#else +#define HOST_MACHINE IMAGE_FILE_MACHINE_I386 +#endif + #include "MemoryModule.h" +struct ExportNameEntry { + LPCSTR name; + WORD idx; +}; + +typedef BOOL (WINAPI *DllEntryProc)(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpReserved); +typedef int (WINAPI *ExeEntryProc)(void); + +#ifdef _WIN64 +typedef struct POINTER_LIST { + struct POINTER_LIST *next; + void *address; +} POINTER_LIST; +#endif + typedef struct { PIMAGE_NT_HEADERS headers; unsigned char *codeBase; HCUSTOMMODULE *modules; int numModules; - int initialized; + BOOL initialized; + BOOL isDLL; + BOOL isRelocated; + CustomAllocFunc alloc; + CustomFreeFunc free; CustomLoadLibraryFunc loadLibrary; CustomGetProcAddressFunc getProcAddress; CustomFreeLibraryFunc freeLibrary; + struct ExportNameEntry *nameExportsTable; void *userdata; + ExeEntryProc exeEntry; + DWORD pageSize; +#ifdef _WIN64 + POINTER_LIST *blockedMemory; +#endif } MEMORYMODULE, *PMEMORYMODULE; -typedef BOOL (WINAPI *DllEntryProc)(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpReserved); +typedef struct { + LPVOID address; + LPVOID alignedAddress; + SIZE_T size; + DWORD characteristics; + BOOL last; +} SECTIONFINALIZEDATA, *PSECTIONFINALIZEDATA; + +#define GET_HEADER_DICTIONARY(module, idx) &(module)->headers->OptionalHeader.DataDirectory[idx] + +static inline uintptr_t +AlignValueDown(uintptr_t value, uintptr_t alignment) { + return value & ~(alignment - 1); +} -#define GET_HEADER_DICTIONARY(module, idx) &(module)->headers->OptionalHeader.DataDirectory[idx] +static inline LPVOID +AlignAddressDown(LPVOID address, uintptr_t alignment) { + return (LPVOID) AlignValueDown((uintptr_t) address, alignment); +} -#ifdef DEBUG_OUTPUT -static void +static inline size_t +AlignValueUp(size_t value, size_t alignment) { + return (value + alignment - 1) & ~(alignment - 1); +} + +static inline void* +OffsetPointer(void* data, ptrdiff_t offset) { + return (void*) ((uintptr_t) data + offset); +} + +static inline void OutputLastError(const char *msg) { +#ifndef DEBUG_OUTPUT + UNREFERENCED_PARAMETER(msg); +#else LPVOID tmp; char *tmpmsg; FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, @@ -78,13 +144,38 @@ OutputLastError(const char *msg) OutputDebugString(tmpmsg); LocalFree(tmpmsg); LocalFree(tmp); -} #endif +} +#ifdef _WIN64 static void -CopySections(const unsigned char *data, PIMAGE_NT_HEADERS old_headers, PMEMORYMODULE module) +FreePointerList(POINTER_LIST *head, CustomFreeFunc freeMemory, void *userdata) { - int i, size; + POINTER_LIST *node = head; + while (node) { + POINTER_LIST *next; + freeMemory(node->address, 0, MEM_RELEASE, userdata); + next = node->next; + free(node); + node = next; + } +} +#endif + +static BOOL +CheckSize(size_t size, size_t expected) { + if (size < expected) { + SetLastError(ERROR_INVALID_DATA); + return FALSE; + } + + return TRUE; +} + +static BOOL +CopySections(const unsigned char *data, size_t size, PIMAGE_NT_HEADERS old_headers, PMEMORYMODULE module) +{ + int i, section_size; unsigned char *codeBase = module->codeBase; unsigned char *dest; PIMAGE_SECTION_HEADER section = IMAGE_FIRST_SECTION(module->headers); @@ -92,29 +183,54 @@ CopySections(const unsigned char *data, PIMAGE_NT_HEADERS old_headers, PMEMORYMO if (section->SizeOfRawData == 0) { // section doesn't contain data in the dll itself, but may define // uninitialized data - size = old_headers->OptionalHeader.SectionAlignment; - if (size > 0) { - dest = (unsigned char *)VirtualAlloc(codeBase + section->VirtualAddress, - size, + section_size = old_headers->OptionalHeader.SectionAlignment; + if (section_size > 0) { + dest = (unsigned char *)module->alloc(codeBase + section->VirtualAddress, + section_size, MEM_COMMIT, - PAGE_READWRITE); + PAGE_READWRITE, + module->userdata); + if (dest == NULL) { + return FALSE; + } - section->Misc.PhysicalAddress = (DWORD) (POINTER_TYPE) dest; - memset(dest, 0, size); + // Always use position from file to support alignments smaller + // than page size (allocation above will align to page size). + dest = codeBase + section->VirtualAddress; + // NOTE: On 64bit systems we truncate to 32bit here but expand + // again later when "PhysicalAddress" is used. + section->Misc.PhysicalAddress = (DWORD) ((uintptr_t) dest & 0xffffffff); + memset(dest, 0, section_size); } // section is empty continue; } + if (!CheckSize(size, section->PointerToRawData + section->SizeOfRawData)) { + return FALSE; + } + // commit memory block and copy data from dll - dest = (unsigned char *)VirtualAlloc(codeBase + section->VirtualAddress, + dest = (unsigned char *)module->alloc(codeBase + section->VirtualAddress, section->SizeOfRawData, MEM_COMMIT, - PAGE_READWRITE); + PAGE_READWRITE, + module->userdata); + if (dest == NULL) { + return FALSE; + } + + // Always use position from file to support alignments smaller + // than page size (allocation above will align to page size). + dest = codeBase + section->VirtualAddress; memcpy(dest, data + section->PointerToRawData, section->SizeOfRawData); - section->Misc.PhysicalAddress = (DWORD) (POINTER_TYPE) dest; + // NOTE: On 64bit systems we truncate to 32bit here but expand + // again later when "PhysicalAddress" is used. + section->Misc.PhysicalAddress = (DWORD) ((uintptr_t) dest & 0xffffffff); } + + return TRUE; } // Protection flags for memory pages (Executable, Readable, Writeable) @@ -130,199 +246,277 @@ static int ProtectionFlags[2][2][2] = { }, }; -static void +static SIZE_T +GetRealSectionSize(PMEMORYMODULE module, PIMAGE_SECTION_HEADER section) { + DWORD size = section->SizeOfRawData; + if (size == 0) { + if (section->Characteristics & IMAGE_SCN_CNT_INITIALIZED_DATA) { + size = module->headers->OptionalHeader.SizeOfInitializedData; + } else if (section->Characteristics & IMAGE_SCN_CNT_UNINITIALIZED_DATA) { + size = module->headers->OptionalHeader.SizeOfUninitializedData; + } + } + return (SIZE_T) size; +} + +static BOOL +FinalizeSection(PMEMORYMODULE module, PSECTIONFINALIZEDATA sectionData) { + DWORD protect, oldProtect; + BOOL executable; + BOOL readable; + BOOL writeable; + + if (sectionData->size == 0) { + return TRUE; + } + + if (sectionData->characteristics & IMAGE_SCN_MEM_DISCARDABLE) { + // section is not needed any more and can safely be freed + if (sectionData->address == sectionData->alignedAddress && + (sectionData->last || + module->headers->OptionalHeader.SectionAlignment == module->pageSize || + (sectionData->size % module->pageSize) == 0) + ) { + // Only allowed to decommit whole pages + module->free(sectionData->address, sectionData->size, MEM_DECOMMIT, module->userdata); + } + return TRUE; + } + + // determine protection flags based on characteristics + executable = (sectionData->characteristics & IMAGE_SCN_MEM_EXECUTE) != 0; + readable = (sectionData->characteristics & IMAGE_SCN_MEM_READ) != 0; + writeable = (sectionData->characteristics & IMAGE_SCN_MEM_WRITE) != 0; + protect = ProtectionFlags[executable][readable][writeable]; + if (sectionData->characteristics & IMAGE_SCN_MEM_NOT_CACHED) { + protect |= PAGE_NOCACHE; + } + + // change memory access flags + if (VirtualProtect(sectionData->address, sectionData->size, protect, &oldProtect) == 0) { + OutputLastError("Error protecting memory page"); + return FALSE; + } + + return TRUE; +} + +static BOOL FinalizeSections(PMEMORYMODULE module) { int i; PIMAGE_SECTION_HEADER section = IMAGE_FIRST_SECTION(module->headers); #ifdef _WIN64 - POINTER_TYPE imageOffset = (module->headers->OptionalHeader.ImageBase & 0xffffffff00000000); + // "PhysicalAddress" might have been truncated to 32bit above, expand to + // 64bits again. + uintptr_t imageOffset = ((uintptr_t) module->headers->OptionalHeader.ImageBase & 0xffffffff00000000); #else - #define imageOffset 0 + static const uintptr_t imageOffset = 0; #endif + SECTIONFINALIZEDATA sectionData; + sectionData.address = (LPVOID)((uintptr_t)section->Misc.PhysicalAddress | imageOffset); + sectionData.alignedAddress = AlignAddressDown(sectionData.address, module->pageSize); + sectionData.size = GetRealSectionSize(module, section); + sectionData.characteristics = section->Characteristics; + sectionData.last = FALSE; + section++; // loop through all sections and change access flags - for (i=0; iheaders->FileHeader.NumberOfSections; i++, section++) { - DWORD protect, oldProtect, size; - int executable = (section->Characteristics & IMAGE_SCN_MEM_EXECUTE) != 0; - int readable = (section->Characteristics & IMAGE_SCN_MEM_READ) != 0; - int writeable = (section->Characteristics & IMAGE_SCN_MEM_WRITE) != 0; - - if (section->Characteristics & IMAGE_SCN_MEM_DISCARDABLE) { - // section is not needed any more and can safely be freed - VirtualFree((LPVOID)((POINTER_TYPE)section->Misc.PhysicalAddress | imageOffset), section->SizeOfRawData, MEM_DECOMMIT); - continue; - } - - // determine protection flags based on characteristics - protect = ProtectionFlags[executable][readable][writeable]; - if (section->Characteristics & IMAGE_SCN_MEM_NOT_CACHED) { - protect |= PAGE_NOCACHE; - } - - // determine size of region - size = section->SizeOfRawData; - if (size == 0) { - if (section->Characteristics & IMAGE_SCN_CNT_INITIALIZED_DATA) { - size = module->headers->OptionalHeader.SizeOfInitializedData; - } else if (section->Characteristics & IMAGE_SCN_CNT_UNINITIALIZED_DATA) { - size = module->headers->OptionalHeader.SizeOfUninitializedData; + for (i=1; iheaders->FileHeader.NumberOfSections; i++, section++) { + LPVOID sectionAddress = (LPVOID)((uintptr_t)section->Misc.PhysicalAddress | imageOffset); + LPVOID alignedAddress = AlignAddressDown(sectionAddress, module->pageSize); + SIZE_T sectionSize = GetRealSectionSize(module, section); + // Combine access flags of all sections that share a page + // TODO(fancycode): We currently share flags of a trailing large section + // with the page of a first small section. This should be optimized. + if (sectionData.alignedAddress == alignedAddress || (uintptr_t) sectionData.address + sectionData.size > (uintptr_t) alignedAddress) { + // Section shares page with previous + if ((section->Characteristics & IMAGE_SCN_MEM_DISCARDABLE) == 0 || (sectionData.characteristics & IMAGE_SCN_MEM_DISCARDABLE) == 0) { + sectionData.characteristics = (sectionData.characteristics | section->Characteristics) & ~IMAGE_SCN_MEM_DISCARDABLE; + } else { + sectionData.characteristics |= section->Characteristics; } + sectionData.size = (((uintptr_t)sectionAddress) + ((uintptr_t) sectionSize)) - (uintptr_t) sectionData.address; + continue; } - if (size > 0) { - // change memory access flags - if (VirtualProtect((LPVOID)((POINTER_TYPE)section->Misc.PhysicalAddress | imageOffset), size, protect, &oldProtect) == 0) { -#ifdef DEBUG_OUTPUT - OutputLastError("Error protecting memory page") -#endif - }; + if (!FinalizeSection(module, §ionData)) { + return FALSE; } + sectionData.address = sectionAddress; + sectionData.alignedAddress = alignedAddress; + sectionData.size = sectionSize; + sectionData.characteristics = section->Characteristics; } -#ifndef _WIN64 -#undef imageOffset -#endif + sectionData.last = TRUE; + if (!FinalizeSection(module, §ionData)) { + return FALSE; + } + return TRUE; } -static void +static BOOL ExecuteTLS(PMEMORYMODULE module) { unsigned char *codeBase = module->codeBase; + PIMAGE_TLS_DIRECTORY tls; + PIMAGE_TLS_CALLBACK* callback; PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_TLS); - if (directory->VirtualAddress > 0) { - PIMAGE_TLS_DIRECTORY tls = (PIMAGE_TLS_DIRECTORY) (codeBase + directory->VirtualAddress); - PIMAGE_TLS_CALLBACK* callback = (PIMAGE_TLS_CALLBACK *) tls->AddressOfCallBacks; - if (callback) { - while (*callback) { - (*callback)((LPVOID) codeBase, DLL_PROCESS_ATTACH, NULL); - callback++; - } + if (directory->VirtualAddress == 0) { + return TRUE; + } + + tls = (PIMAGE_TLS_DIRECTORY) (codeBase + directory->VirtualAddress); + callback = (PIMAGE_TLS_CALLBACK *) tls->AddressOfCallBacks; + if (callback) { + while (*callback) { + (*callback)((LPVOID) codeBase, DLL_PROCESS_ATTACH, NULL); + callback++; } } + return TRUE; } -static void -PerformBaseRelocation(PMEMORYMODULE module, SIZE_T delta) +static BOOL +PerformBaseRelocation(PMEMORYMODULE module, ptrdiff_t delta) { - DWORD i; unsigned char *codeBase = module->codeBase; + PIMAGE_BASE_RELOCATION relocation; PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_BASERELOC); - if (directory->Size > 0) { - PIMAGE_BASE_RELOCATION relocation = (PIMAGE_BASE_RELOCATION) (codeBase + directory->VirtualAddress); - for (; relocation->VirtualAddress > 0; ) { - unsigned char *dest = codeBase + relocation->VirtualAddress; - unsigned short *relInfo = (unsigned short *)((unsigned char *)relocation + IMAGE_SIZEOF_BASE_RELOCATION); - for (i=0; i<((relocation->SizeOfBlock-IMAGE_SIZEOF_BASE_RELOCATION) / 2); i++, relInfo++) { - DWORD *patchAddrHL; -#ifdef _WIN64 - ULONGLONG *patchAddr64; -#endif - int type, offset; + if (directory->Size == 0) { + return (delta == 0); + } - // the upper 4 bits define the type of relocation - type = *relInfo >> 12; - // the lower 12 bits define the offset - offset = *relInfo & 0xfff; + relocation = (PIMAGE_BASE_RELOCATION) (codeBase + directory->VirtualAddress); + for (; relocation->VirtualAddress > 0; ) { + DWORD i; + unsigned char *dest = codeBase + relocation->VirtualAddress; + unsigned short *relInfo = (unsigned short*) OffsetPointer(relocation, IMAGE_SIZEOF_BASE_RELOCATION); + for (i=0; i<((relocation->SizeOfBlock-IMAGE_SIZEOF_BASE_RELOCATION) / 2); i++, relInfo++) { + // the upper 4 bits define the type of relocation + int type = *relInfo >> 12; + // the lower 12 bits define the offset + int offset = *relInfo & 0xfff; + + switch (type) + { + case IMAGE_REL_BASED_ABSOLUTE: + // skip relocation + break; - switch (type) + case IMAGE_REL_BASED_HIGHLOW: + // change complete 32 bit address { - case IMAGE_REL_BASED_ABSOLUTE: - // skip relocation - break; - - case IMAGE_REL_BASED_HIGHLOW: - // change complete 32 bit address - patchAddrHL = (DWORD *) (dest + offset); + DWORD *patchAddrHL = (DWORD *) (dest + offset); *patchAddrHL += (DWORD) delta; - break; + } + break; #ifdef _WIN64 - case IMAGE_REL_BASED_DIR64: - patchAddr64 = (ULONGLONG *) (dest + offset); + case IMAGE_REL_BASED_DIR64: + { + ULONGLONG *patchAddr64 = (ULONGLONG *) (dest + offset); *patchAddr64 += (ULONGLONG) delta; - break; + } + break; #endif - default: - //printf("Unknown relocation: %d\n", type); - break; - } + default: + //printf("Unknown relocation: %d\n", type); + break; } - - // advance to next relocation block - relocation = (PIMAGE_BASE_RELOCATION) (((char *) relocation) + relocation->SizeOfBlock); } + + // advance to next relocation block + relocation = (PIMAGE_BASE_RELOCATION) OffsetPointer(relocation, relocation->SizeOfBlock); } + return TRUE; } -static int +static BOOL BuildImportTable(PMEMORYMODULE module) { - int result=1; unsigned char *codeBase = module->codeBase; - HCUSTOMMODULE *tmp; + PIMAGE_IMPORT_DESCRIPTOR importDesc; + BOOL result = TRUE; PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_IMPORT); - if (directory->Size > 0) { - PIMAGE_IMPORT_DESCRIPTOR importDesc = (PIMAGE_IMPORT_DESCRIPTOR) (codeBase + directory->VirtualAddress); - for (; !IsBadReadPtr(importDesc, sizeof(IMAGE_IMPORT_DESCRIPTOR)) && importDesc->Name; importDesc++) { - POINTER_TYPE *thunkRef; - FARPROC *funcRef; - HCUSTOMMODULE handle = module->loadLibrary((LPCSTR) (codeBase + importDesc->Name), module->userdata); - if (handle == NULL) { - SetLastError(ERROR_MOD_NOT_FOUND); - result = 0; - break; - } + if (directory->Size == 0) { + return TRUE; + } - tmp = (HCUSTOMMODULE *) realloc(module->modules, (module->numModules+1)*(sizeof(HCUSTOMMODULE))); - if (tmp == NULL) { - module->freeLibrary(handle, module->userdata); - SetLastError(ERROR_OUTOFMEMORY); - result = 0; - break; - } - module->modules = tmp; + importDesc = (PIMAGE_IMPORT_DESCRIPTOR) (codeBase + directory->VirtualAddress); + for (; !IsBadReadPtr(importDesc, sizeof(IMAGE_IMPORT_DESCRIPTOR)) && importDesc->Name; importDesc++) { + uintptr_t *thunkRef; + FARPROC *funcRef; + HCUSTOMMODULE *tmp; + HCUSTOMMODULE handle = module->loadLibrary((LPCSTR) (codeBase + importDesc->Name), module->userdata); + if (handle == NULL) { + SetLastError(ERROR_MOD_NOT_FOUND); + result = FALSE; + break; + } - module->modules[module->numModules++] = handle; - if (importDesc->OriginalFirstThunk) { - thunkRef = (POINTER_TYPE *) (codeBase + importDesc->OriginalFirstThunk); - funcRef = (FARPROC *) (codeBase + importDesc->FirstThunk); + tmp = (HCUSTOMMODULE *) realloc(module->modules, (module->numModules+1)*(sizeof(HCUSTOMMODULE))); + if (tmp == NULL) { + module->freeLibrary(handle, module->userdata); + SetLastError(ERROR_OUTOFMEMORY); + result = FALSE; + break; + } + module->modules = tmp; + + module->modules[module->numModules++] = handle; + if (importDesc->OriginalFirstThunk) { + thunkRef = (uintptr_t *) (codeBase + importDesc->OriginalFirstThunk); + funcRef = (FARPROC *) (codeBase + importDesc->FirstThunk); + } else { + // no hint table + thunkRef = (uintptr_t *) (codeBase + importDesc->FirstThunk); + funcRef = (FARPROC *) (codeBase + importDesc->FirstThunk); + } + for (; *thunkRef; thunkRef++, funcRef++) { + if (IMAGE_SNAP_BY_ORDINAL(*thunkRef)) { + *funcRef = module->getProcAddress(handle, (LPCSTR)IMAGE_ORDINAL(*thunkRef), module->userdata); } else { - // no hint table - thunkRef = (POINTER_TYPE *) (codeBase + importDesc->FirstThunk); - funcRef = (FARPROC *) (codeBase + importDesc->FirstThunk); + PIMAGE_IMPORT_BY_NAME thunkData = (PIMAGE_IMPORT_BY_NAME) (codeBase + (*thunkRef)); + *funcRef = module->getProcAddress(handle, (LPCSTR)&thunkData->Name, module->userdata); } - for (; *thunkRef; thunkRef++, funcRef++) { - if (IMAGE_SNAP_BY_ORDINAL(*thunkRef)) { - *funcRef = module->getProcAddress(handle, (LPCSTR)IMAGE_ORDINAL(*thunkRef), module->userdata); - } else { - PIMAGE_IMPORT_BY_NAME thunkData = (PIMAGE_IMPORT_BY_NAME) (codeBase + (*thunkRef)); - *funcRef = module->getProcAddress(handle, (LPCSTR)&thunkData->Name, module->userdata); - } - if (*funcRef == 0) { - result = 0; - break; - } - } - - if (!result) { - module->freeLibrary(handle, module->userdata); - SetLastError(ERROR_PROC_NOT_FOUND); + if (*funcRef == 0) { + result = FALSE; break; } } + + if (!result) { + module->freeLibrary(handle, module->userdata); + SetLastError(ERROR_PROC_NOT_FOUND); + break; + } } return result; } -static HCUSTOMMODULE _LoadLibrary(LPCSTR filename, void *userdata) +LPVOID MemoryDefaultAlloc(LPVOID address, SIZE_T size, DWORD allocationType, DWORD protect, void* userdata) +{ + UNREFERENCED_PARAMETER(userdata); + return VirtualAlloc(address, size, allocationType, protect); +} + +BOOL MemoryDefaultFree(LPVOID lpAddress, SIZE_T dwSize, DWORD dwFreeType, void* userdata) +{ + UNREFERENCED_PARAMETER(userdata); + return VirtualFree(lpAddress, dwSize, dwFreeType); +} + +HCUSTOMMODULE MemoryDefaultLoadLibrary(LPCSTR filename, void *userdata) { - HMODULE result = LoadLibraryA(filename); - (void)userdata; + HMODULE result; + UNREFERENCED_PARAMETER(userdata); + result = LoadLibraryA(filename); if (result == NULL) { return NULL; } @@ -330,105 +524,200 @@ static HCUSTOMMODULE _LoadLibrary(LPCSTR filename, void *userdata) return (HCUSTOMMODULE) result; } -static FARPROC _GetProcAddress(HCUSTOMMODULE module, LPCSTR name, void *userdata) +FARPROC MemoryDefaultGetProcAddress(HCUSTOMMODULE module, LPCSTR name, void *userdata) { - (void)userdata; + UNREFERENCED_PARAMETER(userdata); return (FARPROC) GetProcAddress((HMODULE) module, name); } -static void _FreeLibrary(HCUSTOMMODULE module, void *userdata) +void MemoryDefaultFreeLibrary(HCUSTOMMODULE module, void *userdata) { - (void)userdata; + UNREFERENCED_PARAMETER(userdata); FreeLibrary((HMODULE) module); } -HMEMORYMODULE MemoryLoadLibrary(const void *data) +HMEMORYMODULE MemoryLoadLibrary(const void *data, size_t size) { - return MemoryLoadLibraryEx(data, _LoadLibrary, _GetProcAddress, _FreeLibrary, NULL); + return MemoryLoadLibraryEx(data, size, MemoryDefaultAlloc, MemoryDefaultFree, MemoryDefaultLoadLibrary, MemoryDefaultGetProcAddress, MemoryDefaultFreeLibrary, NULL); } -HMEMORYMODULE MemoryLoadLibraryEx(const void *data, +HMEMORYMODULE MemoryLoadLibraryEx(const void *data, size_t size, + CustomAllocFunc allocMemory, + CustomFreeFunc freeMemory, CustomLoadLibraryFunc loadLibrary, CustomGetProcAddressFunc getProcAddress, CustomFreeLibraryFunc freeLibrary, void *userdata) { - PMEMORYMODULE result; + PMEMORYMODULE result = NULL; PIMAGE_DOS_HEADER dos_header; PIMAGE_NT_HEADERS old_header; unsigned char *code, *headers; - SIZE_T locationDelta; - DllEntryProc DllEntry; - BOOL successfull; + ptrdiff_t locationDelta; + SYSTEM_INFO sysInfo; + PIMAGE_SECTION_HEADER section; + DWORD i; + size_t optionalSectionSize; + size_t lastSectionEnd = 0; + size_t alignedImageSize; +#ifdef _WIN64 + POINTER_LIST *blockedMemory = NULL; +#endif + if (!CheckSize(size, sizeof(IMAGE_DOS_HEADER))) { + return NULL; + } dos_header = (PIMAGE_DOS_HEADER)data; if (dos_header->e_magic != IMAGE_DOS_SIGNATURE) { SetLastError(ERROR_BAD_EXE_FORMAT); return NULL; } + if (!CheckSize(size, dos_header->e_lfanew + sizeof(IMAGE_NT_HEADERS))) { + return NULL; + } old_header = (PIMAGE_NT_HEADERS)&((const unsigned char *)(data))[dos_header->e_lfanew]; if (old_header->Signature != IMAGE_NT_SIGNATURE) { SetLastError(ERROR_BAD_EXE_FORMAT); return NULL; } + if (old_header->FileHeader.Machine != HOST_MACHINE) { + SetLastError(ERROR_BAD_EXE_FORMAT); + return NULL; + } + + if (old_header->OptionalHeader.SectionAlignment & 1) { + // Only support section alignments that are a multiple of 2 + SetLastError(ERROR_BAD_EXE_FORMAT); + return NULL; + } + + section = IMAGE_FIRST_SECTION(old_header); + optionalSectionSize = old_header->OptionalHeader.SectionAlignment; + for (i=0; iFileHeader.NumberOfSections; i++, section++) { + size_t endOfSection; + if (section->SizeOfRawData == 0) { + // Section without data in the DLL + endOfSection = section->VirtualAddress + optionalSectionSize; + } else { + endOfSection = section->VirtualAddress + section->SizeOfRawData; + } + + if (endOfSection > lastSectionEnd) { + lastSectionEnd = endOfSection; + } + } + + GetNativeSystemInfo(&sysInfo); + alignedImageSize = AlignValueUp(old_header->OptionalHeader.SizeOfImage, sysInfo.dwPageSize); + if (alignedImageSize != AlignValueUp(lastSectionEnd, sysInfo.dwPageSize)) { + SetLastError(ERROR_BAD_EXE_FORMAT); + return NULL; + } + // reserve memory for image of library // XXX: is it correct to commit the complete memory region at once? // calling DllEntry raises an exception if we don't... - code = (unsigned char *)VirtualAlloc((LPVOID)(old_header->OptionalHeader.ImageBase), - old_header->OptionalHeader.SizeOfImage, + code = (unsigned char *)allocMemory((LPVOID)(old_header->OptionalHeader.ImageBase), + alignedImageSize, MEM_RESERVE | MEM_COMMIT, - PAGE_READWRITE); + PAGE_READWRITE, + userdata); if (code == NULL) { // try to allocate memory at arbitrary position - code = (unsigned char *)VirtualAlloc(NULL, - old_header->OptionalHeader.SizeOfImage, + code = (unsigned char *)allocMemory(NULL, + alignedImageSize, MEM_RESERVE | MEM_COMMIT, - PAGE_READWRITE); + PAGE_READWRITE, + userdata); if (code == NULL) { SetLastError(ERROR_OUTOFMEMORY); return NULL; } } - result = (PMEMORYMODULE)HeapAlloc(GetProcessHeap(), 0, sizeof(MEMORYMODULE)); +#ifdef _WIN64 + // Memory block may not span 4 GB boundaries. + while ((((uintptr_t) code) >> 32) < (((uintptr_t) (code + alignedImageSize)) >> 32)) { + POINTER_LIST *node = (POINTER_LIST*) malloc(sizeof(POINTER_LIST)); + if (!node) { + freeMemory(code, 0, MEM_RELEASE, userdata); + FreePointerList(blockedMemory, freeMemory, userdata); + SetLastError(ERROR_OUTOFMEMORY); + return NULL; + } + + node->next = blockedMemory; + node->address = code; + blockedMemory = node; + + code = (unsigned char *)allocMemory(NULL, + alignedImageSize, + MEM_RESERVE | MEM_COMMIT, + PAGE_READWRITE, + userdata); + if (code == NULL) { + FreePointerList(blockedMemory, freeMemory, userdata); + SetLastError(ERROR_OUTOFMEMORY); + return NULL; + } + } +#endif + + result = (PMEMORYMODULE)HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, sizeof(MEMORYMODULE)); if (result == NULL) { + freeMemory(code, 0, MEM_RELEASE, userdata); +#ifdef _WIN64 + FreePointerList(blockedMemory, freeMemory, userdata); +#endif SetLastError(ERROR_OUTOFMEMORY); - VirtualFree(code, 0, MEM_RELEASE); return NULL; } result->codeBase = code; - result->numModules = 0; - result->modules = NULL; - result->initialized = 0; + result->isDLL = (old_header->FileHeader.Characteristics & IMAGE_FILE_DLL) != 0; + result->alloc = allocMemory; + result->free = freeMemory; result->loadLibrary = loadLibrary; result->getProcAddress = getProcAddress; result->freeLibrary = freeLibrary; result->userdata = userdata; + result->pageSize = sysInfo.dwPageSize; +#ifdef _WIN64 + result->blockedMemory = blockedMemory; +#endif + + if (!CheckSize(size, old_header->OptionalHeader.SizeOfHeaders)) { + goto error; + } // commit memory for headers - headers = (unsigned char *)VirtualAlloc(code, + headers = (unsigned char *)allocMemory(code, old_header->OptionalHeader.SizeOfHeaders, MEM_COMMIT, - PAGE_READWRITE); + PAGE_READWRITE, + userdata); // copy PE header to code memcpy(headers, dos_header, old_header->OptionalHeader.SizeOfHeaders); result->headers = (PIMAGE_NT_HEADERS)&((const unsigned char *)(headers))[dos_header->e_lfanew]; // update position - result->headers->OptionalHeader.ImageBase = (POINTER_TYPE)code; + result->headers->OptionalHeader.ImageBase = (uintptr_t)code; // copy sections from DLL file block to new memory location - CopySections((const unsigned char *) data, old_header, result); + if (!CopySections((const unsigned char *) data, size, old_header, result)) { + goto error; + } // adjust base address of imported data - locationDelta = (SIZE_T)(code - old_header->OptionalHeader.ImageBase); + locationDelta = (ptrdiff_t)(result->headers->OptionalHeader.ImageBase - old_header->OptionalHeader.ImageBase); if (locationDelta != 0) { - PerformBaseRelocation(result, locationDelta); + result->isRelocated = PerformBaseRelocation(result, locationDelta); + } else { + result->isRelocated = TRUE; } // load required dlls and adjust function table of imports @@ -438,21 +727,31 @@ HMEMORYMODULE MemoryLoadLibraryEx(const void *data, // mark memory pages depending on section headers and release // sections that are marked as "discardable" - FinalizeSections(result); + if (!FinalizeSections(result)) { + goto error; + } // TLS callbacks are executed BEFORE the main loading - ExecuteTLS(result); + if (!ExecuteTLS(result)) { + goto error; + } // get entry point of loaded library if (result->headers->OptionalHeader.AddressOfEntryPoint != 0) { - DllEntry = (DllEntryProc) (code + result->headers->OptionalHeader.AddressOfEntryPoint); - // notify library about attaching to process - successfull = (*DllEntry)((HINSTANCE)code, DLL_PROCESS_ATTACH, 0); - if (!successfull) { - SetLastError(ERROR_DLL_INIT_FAILED); - goto error; + if (result->isDLL) { + DllEntryProc DllEntry = (DllEntryProc)(LPVOID)(code + result->headers->OptionalHeader.AddressOfEntryPoint); + // notify library about attaching to process + BOOL successfull = (*DllEntry)((HINSTANCE)code, DLL_PROCESS_ATTACH, 0); + if (!successfull) { + SetLastError(ERROR_DLL_INIT_FAILED); + goto error; + } + result->initialized = TRUE; + } else { + result->exeEntry = (ExeEntryProc)(LPVOID)(code + result->headers->OptionalHeader.AddressOfEntryPoint); } - result->initialized = 1; + } else { + result->exeEntry = NULL; } return (HMEMORYMODULE)result; @@ -463,14 +762,27 @@ HMEMORYMODULE MemoryLoadLibraryEx(const void *data, return NULL; } -FARPROC MemoryGetProcAddress(HMEMORYMODULE module, LPCSTR name) +static int _compare(const void *a, const void *b) { - unsigned char *codeBase = ((PMEMORYMODULE)module)->codeBase; - int idx=-1; - DWORD i, *nameRef; - WORD *ordinal; + const struct ExportNameEntry *p1 = (const struct ExportNameEntry*) a; + const struct ExportNameEntry *p2 = (const struct ExportNameEntry*) b; + return strcmp(p1->name, p2->name); +} + +static int _find(const void *a, const void *b) +{ + LPCSTR *name = (LPCSTR *) a; + const struct ExportNameEntry *p = (const struct ExportNameEntry*) b; + return strcmp(*name, p->name); +} + +FARPROC MemoryGetProcAddress(HMEMORYMODULE mod, LPCSTR name) +{ + PMEMORYMODULE module = (PMEMORYMODULE)mod; + unsigned char *codeBase = module->codeBase; + DWORD idx = 0; PIMAGE_EXPORT_DIRECTORY exports; - PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY((PMEMORYMODULE)module, IMAGE_DIRECTORY_ENTRY_EXPORT); + PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_EXPORT); if (directory->Size == 0) { // no export table found SetLastError(ERROR_PROC_NOT_FOUND); @@ -484,63 +796,110 @@ FARPROC MemoryGetProcAddress(HMEMORYMODULE module, LPCSTR name) return NULL; } - // search function name in list of exported names - nameRef = (DWORD *) (codeBase + exports->AddressOfNames); - ordinal = (WORD *) (codeBase + exports->AddressOfNameOrdinals); - for (i=0; iNumberOfNames; i++, nameRef++, ordinal++) { - if (_stricmp(name, (const char *) (codeBase + (*nameRef))) == 0) { - idx = *ordinal; - break; + if (HIWORD(name) == 0) { + // load function by ordinal value + if (LOWORD(name) < exports->Base) { + SetLastError(ERROR_PROC_NOT_FOUND); + return NULL; } - } - if (idx == -1) { - // exported symbol not found + idx = LOWORD(name) - exports->Base; + } else if (!exports->NumberOfNames) { SetLastError(ERROR_PROC_NOT_FOUND); return NULL; + } else { + const struct ExportNameEntry *found; + + // Lazily build name table and sort it by names + if (!module->nameExportsTable) { + DWORD i; + DWORD *nameRef = (DWORD *) (codeBase + exports->AddressOfNames); + WORD *ordinal = (WORD *) (codeBase + exports->AddressOfNameOrdinals); + struct ExportNameEntry *entry = (struct ExportNameEntry*) malloc(exports->NumberOfNames * sizeof(struct ExportNameEntry)); + module->nameExportsTable = entry; + if (!entry) { + SetLastError(ERROR_OUTOFMEMORY); + return NULL; + } + for (i=0; iNumberOfNames; i++, nameRef++, ordinal++, entry++) { + entry->name = (const char *) (codeBase + (*nameRef)); + entry->idx = *ordinal; + } + qsort(module->nameExportsTable, + exports->NumberOfNames, + sizeof(struct ExportNameEntry), _compare); + } + + // search function name in list of exported names with binary search + found = (const struct ExportNameEntry*) bsearch(&name, + module->nameExportsTable, + exports->NumberOfNames, + sizeof(struct ExportNameEntry), _find); + if (!found) { + // exported symbol not found + SetLastError(ERROR_PROC_NOT_FOUND); + return NULL; + } + + idx = found->idx; } - if ((DWORD)idx > exports->NumberOfFunctions) { + if (idx > exports->NumberOfFunctions) { // name <-> ordinal number don't match SetLastError(ERROR_PROC_NOT_FOUND); return NULL; } // AddressOfFunctions contains the RVAs to the "real" functions - return (FARPROC) (codeBase + (*(DWORD *) (codeBase + exports->AddressOfFunctions + (idx*4)))); + return (FARPROC)(LPVOID)(codeBase + (*(DWORD *) (codeBase + exports->AddressOfFunctions + (idx*4)))); } void MemoryFreeLibrary(HMEMORYMODULE mod) { - int i; PMEMORYMODULE module = (PMEMORYMODULE)mod; - if (module != NULL) { - if (module->initialized != 0) { - // notify library about detaching from process - DllEntryProc DllEntry = (DllEntryProc) (module->codeBase + module->headers->OptionalHeader.AddressOfEntryPoint); - (*DllEntry)((HINSTANCE)module->codeBase, DLL_PROCESS_DETACH, 0); - module->initialized = 0; - } + if (module == NULL) { + return; + } + if (module->initialized) { + // notify library about detaching from process + DllEntryProc DllEntry = (DllEntryProc)(LPVOID)(module->codeBase + module->headers->OptionalHeader.AddressOfEntryPoint); + (*DllEntry)((HINSTANCE)module->codeBase, DLL_PROCESS_DETACH, 0); + } - if (module->modules != NULL) { - // free previously opened libraries - for (i=0; inumModules; i++) { - if (module->modules[i] != NULL) { - module->freeLibrary(module->modules[i], module->userdata); - } + free(module->nameExportsTable); + if (module->modules != NULL) { + // free previously opened libraries + int i; + for (i=0; inumModules; i++) { + if (module->modules[i] != NULL) { + module->freeLibrary(module->modules[i], module->userdata); } - - free(module->modules); } - if (module->codeBase != NULL) { - // release memory of library - VirtualFree(module->codeBase, 0, MEM_RELEASE); - } + free(module->modules); + } - HeapFree(GetProcessHeap(), 0, module); + if (module->codeBase != NULL) { + // release memory of library + module->free(module->codeBase, 0, MEM_RELEASE, module->userdata); } + +#ifdef _WIN64 + FreePointerList(module->blockedMemory, module->free, module->userdata); +#endif + HeapFree(GetProcessHeap(), 0, module); +} + +int MemoryCallEntryPoint(HMEMORYMODULE mod) +{ + PMEMORYMODULE module = (PMEMORYMODULE)mod; + + if (module == NULL || module->isDLL || module->exeEntry == NULL || !module->isRelocated) { + return -1; + } + + return module->exeEntry(); } #define DEFAULT_LANGUAGE MAKELANGID(LANG_NEUTRAL, SUBLANG_NEUTRAL) @@ -564,11 +923,7 @@ static PIMAGE_RESOURCE_DIRECTORY_ENTRY _MemorySearchResourceEntry( if (!IS_INTRESOURCE(key) && key[0] == TEXT('#')) { // special case: resource id given as string TCHAR *endpos = NULL; -#if defined(UNICODE) - long int tmpkey = (WORD) wcstol((TCHAR *) &key[1], &endpos, 10); -#else - long int tmpkey = (WORD) strtol((TCHAR *) &key[1], &endpos, 10); -#endif + long int tmpkey = (WORD) _tcstol((TCHAR *) &key[1], &endpos, 10); if (tmpkey <= 0xffff && lstrlen(endpos) == 0) { key = MAKEINTRESOURCE(tmpkey); } @@ -578,7 +933,7 @@ static PIMAGE_RESOURCE_DIRECTORY_ENTRY _MemorySearchResourceEntry( // followed by an ordered list of id entries - we can do // a binary search to find faster... if (IS_INTRESOURCE(key)) { - WORD check = (WORD) (POINTER_TYPE) key; + WORD check = (WORD) (uintptr_t) key; start = resources->NumberOfNamedEntries; end = start + resources->NumberOfIdEntries; @@ -596,32 +951,49 @@ static PIMAGE_RESOURCE_DIRECTORY_ENTRY _MemorySearchResourceEntry( } } } else { -#if !defined(UNICODE) - char *searchKey = NULL; - int searchKeyLength = 0; + LPCWSTR searchKey; + size_t searchKeyLen = _tcslen(key); +#if defined(UNICODE) + searchKey = key; +#else + // Resource names are always stored using 16bit characters, need to + // convert string we search for. +#define MAX_LOCAL_KEY_LENGTH 2048 + // In most cases resource names are short, so optimize for that by + // using a pre-allocated array. + wchar_t _searchKeySpace[MAX_LOCAL_KEY_LENGTH+1]; + LPWSTR _searchKey; + if (searchKeyLen > MAX_LOCAL_KEY_LENGTH) { + size_t _searchKeySize = (searchKeyLen + 1) * sizeof(wchar_t); + _searchKey = (LPWSTR) malloc(_searchKeySize); + if (_searchKey == NULL) { + SetLastError(ERROR_OUTOFMEMORY); + return NULL; + } + } else { + _searchKey = &_searchKeySpace[0]; + } + + mbstowcs(_searchKey, key, searchKeyLen); + _searchKey[searchKeyLen] = 0; + searchKey = _searchKey; #endif start = 0; - end = resources->NumberOfIdEntries; + end = resources->NumberOfNamedEntries; while (end > start) { - // resource names are always stored using 16bit characters int cmp; - PIMAGE_RESOURCE_DIR_STRING_U resourceString; + PIMAGE_RESOURCE_DIR_STRING_U resourceString; middle = (start + end) >> 1; - resourceString = (PIMAGE_RESOURCE_DIR_STRING_U) (((char *) root) + (entries[middle].Name & 0x7FFFFFFF)); -#if !defined(UNICODE) - if (searchKey == NULL || searchKeyLength < resourceString->Length) { - void *tmp = realloc(searchKey, resourceString->Length); - if (tmp == NULL) { - break; + resourceString = (PIMAGE_RESOURCE_DIR_STRING_U) OffsetPointer(root, entries[middle].Name & 0x7FFFFFFF); + cmp = _wcsnicmp(searchKey, resourceString->NameString, resourceString->Length); + if (cmp == 0) { + // Handle partial match + if (searchKeyLen > resourceString->Length) { + cmp = 1; + } else if (searchKeyLen < resourceString->Length) { + cmp = -1; } - - searchKey = (char *) tmp; } - wcstombs(searchKey, resourceString->NameString, resourceString->Length); - cmp = strncmp(key, searchKey, resourceString->Length); -#else - cmp = wcsncmp(key, resourceString->NameString, resourceString->Length); -#endif if (cmp < 0) { end = (middle != end ? middle : middle-1); } else if (cmp > 0) { @@ -632,11 +1004,13 @@ static PIMAGE_RESOURCE_DIRECTORY_ENTRY _MemorySearchResourceEntry( } } #if !defined(UNICODE) - free(searchKey); + if (searchKeyLen > MAX_LOCAL_KEY_LENGTH) { + free(_searchKey); + } +#undef MAX_LOCAL_KEY_LENGTH #endif } - return result; } @@ -680,7 +1054,7 @@ HMEMORYRSRC MemoryFindResourceEx(HMEMORYMODULE module, LPCTSTR name, LPCTSTR typ } nameResources = (PIMAGE_RESOURCE_DIRECTORY) (codeBase + directory->VirtualAddress + (foundName->OffsetToData & 0x7fffffff)); - foundLanguage = _MemorySearchResourceEntry(rootResources, nameResources, (LPCTSTR) (POINTER_TYPE) language); + foundLanguage = _MemorySearchResourceEntry(rootResources, nameResources, (LPCTSTR) (uintptr_t) language); if (foundLanguage == NULL) { // requested language not found, use first available if (nameResources->NumberOfIdEntries == 0) { @@ -696,8 +1070,12 @@ HMEMORYRSRC MemoryFindResourceEx(HMEMORYMODULE module, LPCTSTR name, LPCTSTR typ DWORD MemorySizeofResource(HMEMORYMODULE module, HMEMORYRSRC resource) { - PIMAGE_RESOURCE_DATA_ENTRY entry = (PIMAGE_RESOURCE_DATA_ENTRY) resource; - (void)module; + PIMAGE_RESOURCE_DATA_ENTRY entry; + UNREFERENCED_PARAMETER(module); + entry = (PIMAGE_RESOURCE_DATA_ENTRY) resource; + if (entry == NULL) { + return 0; + } return entry->Size; } @@ -706,6 +1084,9 @@ LPVOID MemoryLoadResource(HMEMORYMODULE module, HMEMORYRSRC resource) { unsigned char *codeBase = ((PMEMORYMODULE) module)->codeBase; PIMAGE_RESOURCE_DATA_ENTRY entry = (PIMAGE_RESOURCE_DATA_ENTRY) resource; + if (entry == NULL) { + return NULL; + } return codeBase + entry->OffsetToData; } @@ -719,9 +1100,9 @@ MemoryLoadString(HMEMORYMODULE module, UINT id, LPTSTR buffer, int maxsize) int MemoryLoadStringEx(HMEMORYMODULE module, UINT id, LPTSTR buffer, int maxsize, WORD language) { - HMEMORYRSRC resource; - PIMAGE_RESOURCE_DIR_STRING_U data; - DWORD size; + HMEMORYRSRC resource; + PIMAGE_RESOURCE_DIR_STRING_U data; + DWORD size; if (maxsize == 0) { return 0; } @@ -735,7 +1116,7 @@ MemoryLoadStringEx(HMEMORYMODULE module, UINT id, LPTSTR buffer, int maxsize, WO data = (PIMAGE_RESOURCE_DIR_STRING_U) MemoryLoadResource(module, resource); id = id & 0x0f; while (id--) { - data = (PIMAGE_RESOURCE_DIR_STRING_U) (((char *) data) + (data->Length + 1) * sizeof(WCHAR)); + data = (PIMAGE_RESOURCE_DIR_STRING_U) OffsetPointer(data, (data->Length + 1) * sizeof(WCHAR)); } if (data->Length == 0) { SetLastError(ERROR_RESOURCE_NAME_NOT_FOUND); @@ -756,3 +1137,66 @@ MemoryLoadStringEx(HMEMORYMODULE module, UINT id, LPTSTR buffer, int maxsize, WO #endif return size; } + +#ifdef TESTSUITE +#include + +#ifndef PRIxPTR +#ifdef _WIN64 +#define PRIxPTR "I64x" +#else +#define PRIxPTR "x" +#endif +#endif + +static const uintptr_t AlignValueDownTests[][3] = { + {16, 16, 16}, + {17, 16, 16}, + {32, 16, 32}, + {33, 16, 32}, +#ifdef _WIN64 + {0x12345678abcd1000, 0x1000, 0x12345678abcd1000}, + {0x12345678abcd101f, 0x1000, 0x12345678abcd1000}, +#endif + {0, 0, 0}, +}; + +static const uintptr_t AlignValueUpTests[][3] = { + {16, 16, 16}, + {17, 16, 32}, + {32, 16, 32}, + {33, 16, 48}, +#ifdef _WIN64 + {0x12345678abcd1000, 0x1000, 0x12345678abcd1000}, + {0x12345678abcd101f, 0x1000, 0x12345678abcd2000}, +#endif + {0, 0, 0}, +}; + +BOOL MemoryModuleTestsuite() { + BOOL success = TRUE; + size_t idx; + for (idx = 0; AlignValueDownTests[idx][0]; ++idx) { + const uintptr_t* tests = AlignValueDownTests[idx]; + uintptr_t value = AlignValueDown(tests[0], tests[1]); + if (value != tests[2]) { + printf("AlignValueDown failed for 0x%" PRIxPTR "/0x%" PRIxPTR ": expected 0x%" PRIxPTR ", got 0x%" PRIxPTR "\n", + tests[0], tests[1], tests[2], value); + success = FALSE; + } + } + for (idx = 0; AlignValueDownTests[idx][0]; ++idx) { + const uintptr_t* tests = AlignValueUpTests[idx]; + uintptr_t value = AlignValueUp(tests[0], tests[1]); + if (value != tests[2]) { + printf("AlignValueUp failed for 0x%" PRIxPTR "/0x%" PRIxPTR ": expected 0x%" PRIxPTR ", got 0x%" PRIxPTR "\n", + tests[0], tests[1], tests[2], value); + success = FALSE; + } + } + if (success) { + printf("OK\n"); + } + return success; +} +#endif diff --git a/win/MemoryModule.h b/win/MemoryModule.h index adeacb6521c..a728f6b1416 100644 --- a/win/MemoryModule.h +++ b/win/MemoryModule.h @@ -2,7 +2,7 @@ * Memory DLL loading code * Version 0.0.4 * - * Copyright (c) 2004-2014 by Joachim Bauch / mail@joachim-bauch.de + * Copyright (c) 2004-2015 by Joachim Bauch / mail@joachim-bauch.de * http://www.joachim-bauch.de * * The contents of this file are subject to the Mozilla Public License Version @@ -19,7 +19,7 @@ * * The Initial Developer of the Original Code is Joachim Bauch. * - * Portions created by Joachim Bauch are Copyright (C) 2004-2014 + * Portions created by Joachim Bauch are Copyright (C) 2004-2015 * Joachim Bauch. All Rights Reserved. * */ @@ -39,39 +39,58 @@ typedef void *HCUSTOMMODULE; extern "C" { #endif +typedef LPVOID (*CustomAllocFunc)(LPVOID, SIZE_T, DWORD, DWORD, void*); +typedef BOOL (*CustomFreeFunc)(LPVOID, SIZE_T, DWORD, void*); typedef HCUSTOMMODULE (*CustomLoadLibraryFunc)(LPCSTR, void *); typedef FARPROC (*CustomGetProcAddressFunc)(HCUSTOMMODULE, LPCSTR, void *); typedef void (*CustomFreeLibraryFunc)(HCUSTOMMODULE, void *); /** - * Load DLL from memory location. + * Load EXE/DLL from memory location with the given size. * * All dependencies are resolved using default LoadLibrary/GetProcAddress * calls through the Windows API. */ -HMEMORYMODULE MemoryLoadLibrary(const void *); +HMEMORYMODULE MemoryLoadLibrary(const void *, size_t); /** - * Load DLL from memory location using custom dependency resolvers. + * Load EXE/DLL from memory location with the given size using custom dependency + * resolvers. * * Dependencies will be resolved using passed callback methods. */ -HMEMORYMODULE MemoryLoadLibraryEx(const void *, +HMEMORYMODULE MemoryLoadLibraryEx(const void *, size_t, + CustomAllocFunc, + CustomFreeFunc, CustomLoadLibraryFunc, CustomGetProcAddressFunc, CustomFreeLibraryFunc, void *); /** - * Get address of exported method. + * Get address of exported method. Supports loading both by name and by + * ordinal value. */ FARPROC MemoryGetProcAddress(HMEMORYMODULE, LPCSTR); /** - * Free previously loaded DLL. + * Free previously loaded EXE/DLL. */ void MemoryFreeLibrary(HMEMORYMODULE); +/** + * Execute entry point (EXE only). The entry point can only be executed + * if the EXE has been loaded to the correct base address or it could + * be relocated (i.e. relocation information have not been stripped by + * the linker). + * + * Important: calling this function will not return, i.e. once the loaded + * EXE finished running, the process will terminate. + * + * Returns a negative value if the entry point could not be executed. + */ +int MemoryCallEntryPoint(HMEMORYMODULE); + /** * Find the location of a resource with the specified type and name. */ @@ -102,6 +121,46 @@ int MemoryLoadString(HMEMORYMODULE, UINT, LPTSTR, int); */ int MemoryLoadStringEx(HMEMORYMODULE, UINT, LPTSTR, int, WORD); +/** +* Default implementation of CustomAllocFunc that calls VirtualAlloc +* internally to allocate memory for a library +* +* This is the default as used by MemoryLoadLibrary. +*/ +LPVOID MemoryDefaultAlloc(LPVOID, SIZE_T, DWORD, DWORD, void *); + +/** +* Default implementation of CustomFreeFunc that calls VirtualFree +* internally to free the memory used by a library +* +* This is the default as used by MemoryLoadLibrary. +*/ +BOOL MemoryDefaultFree(LPVOID, SIZE_T, DWORD, void *); + +/** + * Default implementation of CustomLoadLibraryFunc that calls LoadLibraryA + * internally to load an additional libary. + * + * This is the default as used by MemoryLoadLibrary. + */ +HCUSTOMMODULE MemoryDefaultLoadLibrary(LPCSTR, void *); + +/** + * Default implementation of CustomGetProcAddressFunc that calls GetProcAddress + * internally to get the address of an exported function. + * + * This is the default as used by MemoryLoadLibrary. + */ +FARPROC MemoryDefaultGetProcAddress(HCUSTOMMODULE, LPCSTR, void *); + +/** + * Default implementation of CustomFreeLibraryFunc that calls FreeLibrary + * internally to release an additional libary. + * + * This is the default as used by MemoryLoadLibrary. + */ +void MemoryDefaultFreeLibrary(HCUSTOMMODULE, void *); + #ifdef __cplusplus } #endif diff --git a/win/makefile.vc b/win/makefile.vc index f48aa853275..42ac244a1fc 100644 --- a/win/makefile.vc +++ b/win/makefile.vc @@ -911,7 +911,7 @@ $(TMP_DIR)\tclZlib.obj: $(GENERICDIR)\tclZlib.c $(cc32) $(pkgcflags) -I$(COMPATDIR)\zlib -Fo$@ $? $(TMP_DIR)\MemoryModule.obj: $(WIN_DIR)\MemoryModule.c - $(cc32) $(pkgcflags) /DUNICODE -Fo$@ $? + $(cc32) $(pkgcflags) /DUNICODE /D_UNICODE -Fo$@ $? # Following the lead of the autoconf based make, we define the # CFG_RUNTIME_*DIR flags specifically for tclPkgConfig diff --git a/win/tclWinLoad.c b/win/tclWinLoad.c index d15ead51897..75e6d4e289c 100644 --- a/win/tclWinLoad.c +++ b/win/tclWinLoad.c @@ -441,7 +441,7 @@ TclpLoadMemory( Tcl_Interp *interp, /* Used for error reporting. */ void *data, /* Buffer containing the desired code * (allocated with TclpLoadMemoryGetBuffer). */ - TCL_UNUSED(size_t), /* Allocation size of buffer. */ + size_t size, /* Allocation size of buffer. */ Tcl_Size codeSize, /* Size of code data read into buffer or TCL_INDEX_NONE * if an error occurred and buffer should be * free'd. */ @@ -461,7 +461,7 @@ TclpLoadMemory( Tcl_Free(data); return TCL_ERROR; } - hInstance = MemoryLoadLibrary(data); + hInstance = MemoryLoadLibrary(data, size); if (hInstance == NULL) { Tcl_SetObjResult(interp, Tcl_NewStringObj( "cannot load dll in memory", TCL_INDEX_NONE)); From 485d769ffcac48d55879b8c7f5f078af0e580ad0 Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Sat, 23 Nov 2024 16:29:03 +0000 Subject: [PATCH 08/33] Fix [https://github.com/fancycode/MemoryModule/pull/96|MemoryGetProcAddress by ordinal value] (not used by Tcl) --- win/MemoryModule.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/win/MemoryModule.c b/win/MemoryModule.c index f6b2fe7ff7b..3603bee8d32 100644 --- a/win/MemoryModule.c +++ b/win/MemoryModule.c @@ -790,7 +790,9 @@ FARPROC MemoryGetProcAddress(HMEMORYMODULE mod, LPCSTR name) } exports = (PIMAGE_EXPORT_DIRECTORY) (codeBase + directory->VirtualAddress); - if (exports->NumberOfNames == 0 || exports->NumberOfFunctions == 0) { + //if (exports->NumberOfNames == 0 || exports->NumberOfFunctions == 0) { + /*2019.12.24.zoand*/ + if (exports->NumberOfNames == 0 && exports->NumberOfFunctions == 0) { // DLL doesn't export anything SetLastError(ERROR_PROC_NOT_FOUND); return NULL; From 5455f01e21a9f4a9327c169f09eb3f1f3f736984 Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Sat, 23 Nov 2024 16:51:10 +0000 Subject: [PATCH 09/33] Re-apply [b11c0b7e61], for testing purposes only --- win/Makefile.in | 9 +++++++-- win/makefile.vc | 9 +++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/win/Makefile.in b/win/Makefile.in index be9d79565d2..ac756c72093 100644 --- a/win/Makefile.in +++ b/win/Makefile.in @@ -550,8 +550,13 @@ ${TCL_ZIP_FILE}: ${ZIP_INSTALL_OBJS} ${DDE_DLL_FILE} ${REG_DLL_FILE} @( \ $(COPY) -a $(TOP_DIR)/library/* ${TCL_VFS_PATH}; \ $(COPY) -a ${TCL_VFS_PATH}/manifest.txt ${TCL_VFS_PATH}/pkgIndex.tcl; \ - rm -rf ${TCL_VFS_PATH}/dde; \ - rm -rf ${TCL_VFS_PATH}/registry; \ + if test "${ZIPFS_BUILD}" != "2" ; then \ + $(COPY) ${DDE_DLL_FILE} ${TCL_VFS_PATH}/dde; \ + $(COPY) ${REG_DLL_FILE} ${TCL_VFS_PATH}/registry; \ + else \ + rm -rf ${TCL_VFS_PATH}/dde; \ + rm -rf ${TCL_VFS_PATH}/registry; \ + fi \ ) (zip=`(realpath '${NATIVE_ZIP}' || readlink -m '${NATIVE_ZIP}') 2>/dev/null || \ (echo '${NATIVE_ZIP}' | sed "s?^\./?$$(pwd)/?")`; \ diff --git a/win/makefile.vc b/win/makefile.vc index 42ac244a1fc..6d5bfb0be5a 100644 --- a/win/makefile.vc +++ b/win/makefile.vc @@ -662,11 +662,16 @@ $(TCLSCRIPTZIP): $(TCLLIB) $(TCLSH) dlls @$(MKDIR) "$(LIBTCLVFS)" @$(CPYDIR) $(LIBDIR) "$(LIBTCLVFS)\tcl_library" @move /y "$(LIBTCLVFS)\tcl_library\manifest.txt" "$(LIBTCLVFS)\tcl_library\pkgIndex.tcl" > NUL +!if $(STATIC_BUILD) # Remove the registry and dde directories as the DLLS are still external @del "$(LIBTCLVFS)\tcl_library\registry\pkgIndex.tcl" @rmdir "$(LIBTCLVFS)\tcl_library\registry" @del "$(LIBTCLVFS)\tcl_library\dde\pkgIndex.tcl" @rmdir "$(LIBTCLVFS)\tcl_library\dde" +!else + @$(COPY) $(TCLDDELIB) "$(LIBTCLVFS)\tcl_library\dde + @$(COPY) $(TCLREGLIB) "$(LIBTCLVFS)\tcl_library\registry +!endif @echo cd {$(OUT_DIR)} > "$(OUT_DIR)\zipper.tcl" @echo file delete -force {$(@F)} >> "$(OUT_DIR)\zipper.tcl" @echo zipfs mkzip {$(@F)} {$(LIBTCLVFSSUBDIR)} {$(LIBTCLVFSSUBDIR)} >> "$(OUT_DIR)\zipper.tcl" @@ -1128,13 +1133,17 @@ install-libraries: tclConfig tcl-nmake install-msgs install-tzdata "$(MODULE_INSTALL_DIR)\9.0\platform\shell-$(PKG_SHELL_VER).tm" !endif @echo Installing $(TCLDDELIBNAME) +!if !$(STATIC_BUILD) @$(CPY) "$(TCLDDELIB)" "$(LIB_INSTALL_DIR)\dde$(DDEDOTVERSION)\" @$(CPY) "$(ROOT)\library\dde\pkgIndex.tcl" \ "$(LIB_INSTALL_DIR)\dde$(DDEDOTVERSION)\" +!endif @echo Installing $(TCLREGLIBNAME) +!if !$(STATIC_BUILD) @$(CPY) "$(TCLREGLIB)" "$(LIB_INSTALL_DIR)\registry$(REGDOTVERSION)\" @$(CPY) "$(ROOT)\library\registry\pkgIndex.tcl" \ "$(LIB_INSTALL_DIR)\registry$(REGDOTVERSION)\" +!endif !if !$(TCL_EMBED_SCRIPTS) @echo Installing encodings @$(CPY) "$(ROOT)\library\encoding\*.enc" \ From c83978fab2598c0b20ddb2025bd8f2c93e426df5 Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Sun, 24 Nov 2024 21:44:01 +0000 Subject: [PATCH 10/33] Disable copying to /tmp. For test purposes only --- generic/tclIOUtil.c | 1 + 1 file changed, 1 insertion(+) diff --git a/generic/tclIOUtil.c b/generic/tclIOUtil.c index 2028073cb9e..d89f9bcaa59 100644 --- a/generic/tclIOUtil.c +++ b/generic/tclIOUtil.c @@ -3263,6 +3263,7 @@ Tcl_LoadFile( } mustCopyToTempAnyway: + Tcl_Panic("Copying to temp disabled"); // TODO!: Remove this when merging to trunk! #endif /* TCL_LOAD_FROM_MEMORY */ /* From 20ee777ccbe654ba6ffb0c3ae573592a0e29e86c Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Mon, 25 Nov 2024 15:27:52 +0000 Subject: [PATCH 11/33] Fix compilation: TclpLoadMemory doesn't have an "interp" parameter any more --- win/tclWinLoad.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/win/tclWinLoad.c b/win/tclWinLoad.c index 75e6d4e289c..87f6872c059 100644 --- a/win/tclWinLoad.c +++ b/win/tclWinLoad.c @@ -438,7 +438,6 @@ FindMemSymbol( MODULE_SCOPE int TclpLoadMemory( - Tcl_Interp *interp, /* Used for error reporting. */ void *data, /* Buffer containing the desired code * (allocated with TclpLoadMemoryGetBuffer). */ size_t size, /* Allocation size of buffer. */ @@ -463,9 +462,6 @@ TclpLoadMemory( } hInstance = MemoryLoadLibrary(data, size); if (hInstance == NULL) { - Tcl_SetObjResult(interp, Tcl_NewStringObj( - "cannot load dll in memory", TCL_INDEX_NONE)); - Tcl_SetErrorCode(interp, "WIN_LOAD", "LOAD_MEMORY", (char *)NULL); return TCL_ERROR; } handlePtr = (Tcl_LoadHandle)Tcl_Alloc(sizeof(struct Tcl_LoadHandle_)); From 5858dd2a09551d5799db82e840d1a97d3997b532 Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Mon, 25 Nov 2024 15:29:47 +0000 Subject: [PATCH 12/33] Hack: Add a pkgIndex.tcl file and tcl9tk90.dll file to tcl90.dll. For test purposes. --- win/Makefile.in | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/win/Makefile.in b/win/Makefile.in index ac756c72093..81ad41f043a 100644 --- a/win/Makefile.in +++ b/win/Makefile.in @@ -556,7 +556,9 @@ ${TCL_ZIP_FILE}: ${ZIP_INSTALL_OBJS} ${DDE_DLL_FILE} ${REG_DLL_FILE} else \ rm -rf ${TCL_VFS_PATH}/dde; \ rm -rf ${TCL_VFS_PATH}/registry; \ - fi \ + fi; \ + echo " package ifneeded tk 9.0.1 [list load [file normalize [file join \$$dir tcl9tk90.dll]]]" > ${TCL_VFS_ROOT}/pkgIndex.tcl; \ + $(COPY) -a /usr/bin/tcl9tk90.dll ${TCL_VFS_ROOT}; \ ) (zip=`(realpath '${NATIVE_ZIP}' || readlink -m '${NATIVE_ZIP}') 2>/dev/null || \ (echo '${NATIVE_ZIP}' | sed "s?^\./?$$(pwd)/?")`; \ From 6b417e601a4a80e57cb8ad3d37eb2440995f0794 Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Tue, 26 Nov 2024 10:16:08 +0000 Subject: [PATCH 13/33] Experiment with Tk done, undo hack --- win/Makefile.in | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/win/Makefile.in b/win/Makefile.in index 81ad41f043a..ac756c72093 100644 --- a/win/Makefile.in +++ b/win/Makefile.in @@ -556,9 +556,7 @@ ${TCL_ZIP_FILE}: ${ZIP_INSTALL_OBJS} ${DDE_DLL_FILE} ${REG_DLL_FILE} else \ rm -rf ${TCL_VFS_PATH}/dde; \ rm -rf ${TCL_VFS_PATH}/registry; \ - fi; \ - echo " package ifneeded tk 9.0.1 [list load [file normalize [file join \$$dir tcl9tk90.dll]]]" > ${TCL_VFS_ROOT}/pkgIndex.tcl; \ - $(COPY) -a /usr/bin/tcl9tk90.dll ${TCL_VFS_ROOT}; \ + fi \ ) (zip=`(realpath '${NATIVE_ZIP}' || readlink -m '${NATIVE_ZIP}') 2>/dev/null || \ (echo '${NATIVE_ZIP}' | sed "s?^\./?$$(pwd)/?")`; \ From 6f39fd58f6bbc328e408270731d133551bdeb585 Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Tue, 26 Nov 2024 14:34:14 +0000 Subject: [PATCH 14/33] tclWinLoad.c must always be compiled with -D_UNICODE --- win/tclWinLoad.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/win/tclWinLoad.c b/win/tclWinLoad.c index 87f6872c059..76ec743e96d 100644 --- a/win/tclWinLoad.c +++ b/win/tclWinLoad.c @@ -13,8 +13,8 @@ #include "tclWinInt.h" #ifdef TCL_LOAD_FROM_MEMORY -#undef UNICODE -#define UNICODE +#undef _UNICODE +#define _UNICODE #include "MemoryModule.h" #endif From 67d6949c6e32a197e97d95bd605f818dcbe4d919 Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Tue, 26 Nov 2024 21:09:04 +0000 Subject: [PATCH 15/33] fix a [https://github.com/fancycode/MemoryModule/pull/116|bug in PerformBaseRelocation] --- win/MemoryModule.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/win/MemoryModule.c b/win/MemoryModule.c index 3603bee8d32..a0d75448026 100644 --- a/win/MemoryModule.c +++ b/win/MemoryModule.c @@ -382,6 +382,7 @@ static BOOL PerformBaseRelocation(PMEMORYMODULE module, ptrdiff_t delta) { unsigned char *codeBase = module->codeBase; + DWORD relocation_size; PIMAGE_BASE_RELOCATION relocation; PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_BASERELOC); @@ -389,11 +390,16 @@ PerformBaseRelocation(PMEMORYMODULE module, ptrdiff_t delta) return (delta == 0); } + relocation_size = directory->Size; relocation = (PIMAGE_BASE_RELOCATION) (codeBase + directory->VirtualAddress); - for (; relocation->VirtualAddress > 0; ) { + + for (;relocation_size; ) { DWORD i; unsigned char *dest = codeBase + relocation->VirtualAddress; unsigned short *relInfo = (unsigned short*) OffsetPointer(relocation, IMAGE_SIZEOF_BASE_RELOCATION); + + relocation_size -= relocation->SizeOfBlock; + for (i=0; i<((relocation->SizeOfBlock-IMAGE_SIZEOF_BASE_RELOCATION) / 2); i++, relInfo++) { // the upper 4 bits define the type of relocation int type = *relInfo >> 12; From 0a8c62602d49bd26b2bffa3fd0de26d5b933caa3 Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Tue, 26 Nov 2024 21:18:45 +0000 Subject: [PATCH 16/33] fix [https://github.com/fancycode/MemoryModule/pull/94|SectionAlignment] --- win/MemoryModule.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/win/MemoryModule.c b/win/MemoryModule.c index a0d75448026..8c97caeeb1f 100644 --- a/win/MemoryModule.c +++ b/win/MemoryModule.c @@ -560,7 +560,6 @@ HMEMORYMODULE MemoryLoadLibraryEx(const void *data, size_t size, PIMAGE_NT_HEADERS old_header; unsigned char *code, *headers; ptrdiff_t locationDelta; - SYSTEM_INFO sysInfo; PIMAGE_SECTION_HEADER section; DWORD i; size_t optionalSectionSize; @@ -615,9 +614,8 @@ HMEMORYMODULE MemoryLoadLibraryEx(const void *data, size_t size, } } - GetNativeSystemInfo(&sysInfo); - alignedImageSize = AlignValueUp(old_header->OptionalHeader.SizeOfImage, sysInfo.dwPageSize); - if (alignedImageSize != AlignValueUp(lastSectionEnd, sysInfo.dwPageSize)) { + alignedImageSize = AlignValueUp(old_header->OptionalHeader.SizeOfImage, old_header->OptionalHeader.SectionAlignment); + if (alignedImageSize != AlignValueUp(lastSectionEnd, old_header->OptionalHeader.SectionAlignment)) { SetLastError(ERROR_BAD_EXE_FORMAT); return NULL; } @@ -690,7 +688,7 @@ HMEMORYMODULE MemoryLoadLibraryEx(const void *data, size_t size, result->getProcAddress = getProcAddress; result->freeLibrary = freeLibrary; result->userdata = userdata; - result->pageSize = sysInfo.dwPageSize; + result->pageSize = old_header->OptionalHeader.SectionAlignment; #ifdef _WIN64 result->blockedMemory = blockedMemory; #endif From d5f29770b871b209d707124c66ba5a221be94074 Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Tue, 26 Nov 2024 21:41:28 +0000 Subject: [PATCH 17/33] Make build switchable by defining (or not) TCL_LOAD_FROM_MEMORY --- win/Makefile.in | 4 ---- win/makefile.vc | 4 ---- win/tclWinLoad.c | 4 +++- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/win/Makefile.in b/win/Makefile.in index ac756c72093..5b543b7d72d 100644 --- a/win/Makefile.in +++ b/win/Makefile.in @@ -453,7 +453,6 @@ TOMMATH_OBJS = \ WIN_OBJS = \ - MemoryModule.$(OBJEXT) \ tclWin32Dll.$(OBJEXT) \ tclWinChan.$(OBJEXT) \ tclWinConsole.$(OBJEXT) \ @@ -703,9 +702,6 @@ tclZipfs.${OBJEXT}: $(GENERIC_DIR)/tclZipfs.c $(CC) -c $(CC_SWITCHES) -DBUILD_tcl \ $(ZLIB_INCLUDE) -I$(MINIZIP_DIR_NATIVE) @DEPARG@ $(CC_OBJNAME) -MemoryModule.${OBJEXT}: $(WIN_DIR)/MemoryModule.c - $(CC) -c $(CC_SWITCHES) -DUNICODE -D_UNICODE @DEPARG@ $(CC_OBJNAME) - # TIP #59, embedding of configuration information into the binary library. # diff --git a/win/makefile.vc b/win/makefile.vc index 6d5bfb0be5a..10af22eb533 100644 --- a/win/makefile.vc +++ b/win/makefile.vc @@ -429,7 +429,6 @@ TOMMATHOBJS = $(OUT_DIR)\tommath.lib !endif PLATFORMOBJS = \ - $(TMP_DIR)\MemoryModule.obj \ $(TMP_DIR)\tclWin32Dll.obj \ $(TMP_DIR)\tclWinChan.obj \ $(TMP_DIR)\tclWinConsole.obj \ @@ -915,9 +914,6 @@ $(TMP_DIR)\tclZipfs.obj: $(GENERICDIR)\tclZipfs.c $(TMP_DIR)\tclZlib.obj: $(GENERICDIR)\tclZlib.c $(cc32) $(pkgcflags) -I$(COMPATDIR)\zlib -Fo$@ $? -$(TMP_DIR)\MemoryModule.obj: $(WIN_DIR)\MemoryModule.c - $(cc32) $(pkgcflags) /DUNICODE /D_UNICODE -Fo$@ $? - # Following the lead of the autoconf based make, we define the # CFG_RUNTIME_*DIR flags specifically for tclPkgConfig # and not as part of the global defines. These are all defined diff --git a/win/tclWinLoad.c b/win/tclWinLoad.c index 76ec743e96d..ff268273068 100644 --- a/win/tclWinLoad.c +++ b/win/tclWinLoad.c @@ -13,9 +13,11 @@ #include "tclWinInt.h" #ifdef TCL_LOAD_FROM_MEMORY +#undef UNICODE #undef _UNICODE +#define UNICODE #define _UNICODE -#include "MemoryModule.h" +#include "MemoryModule.c" #endif /* From b742bf9f07ccf89d0aac7dde0f04c59d1ad037c2 Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Wed, 27 Nov 2024 10:44:38 +0000 Subject: [PATCH 18/33] Implement build-time switch --- generic/tclIOUtil.c | 2 ++ win/Makefile.in | 5 ++++- win/configure | 23 +++++++++++++++++++++++ win/configure.ac | 12 ++++++++++++ win/makefile.vc | 19 ++++++++++++++++++- win/tclWinLoad.c | 4 ---- 6 files changed, 59 insertions(+), 6 deletions(-) diff --git a/generic/tclIOUtil.c b/generic/tclIOUtil.c index d89f9bcaa59..bd39c4a567e 100644 --- a/generic/tclIOUtil.c +++ b/generic/tclIOUtil.c @@ -3263,7 +3263,9 @@ Tcl_LoadFile( } mustCopyToTempAnyway: +#ifdef _WIN32 Tcl_Panic("Copying to temp disabled"); // TODO!: Remove this when merging to trunk! +#endif #endif /* TCL_LOAD_FROM_MEMORY */ /* diff --git a/win/Makefile.in b/win/Makefile.in index 5b543b7d72d..3f7044cd0d3 100644 --- a/win/Makefile.in +++ b/win/Makefile.in @@ -265,7 +265,7 @@ MINIZIP_OBJS = \ ZIP_INSTALL_OBJS = @ZIP_INSTALL_OBJS@ CC_SWITCHES = -I"${BUILD_DIR}" -I"${GENERIC_DIR_NATIVE}" -I"${TOMMATH_DIR_NATIVE}" \ --I"${ZLIB_DIR_NATIVE}" -I"${WIN_DIR_NATIVE}" -DTCL_LOAD_FROM_MEMORY \ +-I"${ZLIB_DIR_NATIVE}" -I"${WIN_DIR_NATIVE}" \ ${CFLAGS} ${CFLAGS_WARNING} ${SHLIB_CFLAGS} -DMP_PREC=4 \ ${AC_FLAGS} ${COMPILE_DEBUG_FLAGS} ${NO_DEPRECATED_FLAGS} @@ -676,6 +676,9 @@ tclTestMain.${OBJEXT}: tclAppInit.c tclWinInit.${OBJEXT}: tclWinInit.c $(CC) -c $(CC_SWITCHES) -DBUILD_tcl $(EXTFLAGS) @DEPARG@ $(CC_OBJNAME) +tclWinLoad.${OBJEXT}: tclWinLoad.c + $(CC) -c $(CC_SWITCHES) -DBUILD_tcl -DUNICODE -D_UNICODE $(EXTFLAGS) @DEPARG@ $(CC_OBJNAME) + tclWinPipe.${OBJEXT}: tclWinPipe.c $(CC) -c $(CC_SWITCHES) -DBUILD_tcl $(EXTFLAGS) @DEPARG@ $(CC_OBJNAME) diff --git a/win/configure b/win/configure index b45db607c38..f1ff71cefc3 100755 --- a/win/configure +++ b/win/configure @@ -712,6 +712,7 @@ VC_MANIFEST_EMBED_DLL CPP LDFLAGS_DEFAULT CFLAGS_DEFAULT +TCL_LOAD_FROM_MEMORY INSTALL_MSGS INSTALL_LIBRARIES TCL_ZIP_FILE @@ -797,6 +798,7 @@ with_encoding enable_shared enable_64bit enable_zipfs +enable_memorymodule enable_symbols enable_embedded_manifest ' @@ -1430,6 +1432,8 @@ Optional Features: --enable-shared build and link with shared libraries (default: on) --enable-64bit enable 64bit support (where applicable) --enable-zipfs build with Zipfs support (default: on) + --enable-memorymodule load dll's from a vfs in memory in stead of copying + to a temporary directory (default: off) --enable-symbols build with debugging symbols (default: off) --enable-embedded-manifest embed manifest if possible (default: yes) @@ -5262,6 +5266,25 @@ fi +#-------------------------------------------------------------------- +# Zipfs support - Tip 430 +#-------------------------------------------------------------------- +# Check whether --enable-memorymodule was given. +if test ${enable_memorymodule+y} +then : + enableval=$enable_memorymodule; tcl_ok=$enableval +else case e in #( + e) tcl_ok=no ;; +esac +fi + +if test "$tcl_ok" != "no" ; then + +printf "%s\n" "#define TCL_LOAD_FROM_MEMORY 1" >>confdefs.h + +fi + + #-------------------------------------------------------------------- # Perform additinal compiler tests. #-------------------------------------------------------------------- diff --git a/win/configure.ac b/win/configure.ac index 8ad932a8987..42c3b53cdb9 100644 --- a/win/configure.ac +++ b/win/configure.ac @@ -209,6 +209,18 @@ AC_SUBST(INSTALL_LIBRARIES) AC_SUBST(INSTALL_MSGS) +#-------------------------------------------------------------------- +# Zipfs support - Tip 430 +#-------------------------------------------------------------------- +AC_ARG_ENABLE(memorymodule, + AS_HELP_STRING([--enable-memorymodule], + [load dll's from a vfs in memory in stead of copying to a temporary directory (default: off)]), + [tcl_ok=$enableval], [tcl_ok=no]) +if test "$tcl_ok" != "no" ; then + AC_DEFINE(TCL_LOAD_FROM_MEMORY, 1, [Are we building with MemoryModule enabled?]) +fi +AC_SUBST(TCL_LOAD_FROM_MEMORY) + #-------------------------------------------------------------------- # Perform additinal compiler tests. #-------------------------------------------------------------------- diff --git a/win/makefile.vc b/win/makefile.vc index 10af22eb533..f19ba1adf98 100644 --- a/win/makefile.vc +++ b/win/makefile.vc @@ -57,6 +57,8 @@ # Any combination of the above may be used (comma separated). # 'none' will over-ride everything to nothing. # +# memorymodule = load dll's from a vfs in memory in stead of +# copying to a temporary directory. # noembed = Without this option, the Tcl core library scripts # are embedded into the executable if "static" is # specified in OPTS, or into the DLL otherwise. If @@ -165,6 +167,13 @@ VERSION = $(TCL_MAJOR_VERSION)$(TCL_MINOR_VERSION) !endif !endif +!if [nmakehlp -f $(OPTS) "memorymodule"] +!message *** Option memorymodule specified. +TCL_LOAD_FROM_MEMORY = 1 +!else +TCL_LOAD_FROM_MEMORY = 0 +!endif + !if [nmakehlp -f $(OPTS) "noembed"] !message *** Option noembed specified. Tcl script library will not be appended to the binary. TCL_EMBED_SCRIPTS = 0 @@ -472,7 +481,11 @@ LIBTCLVFS = $(OUT_DIR)\$(LIBTCLVFSSUBDIR) # Additional include and C macro definitions for the implicit rules # defined in rules.vc PRJ_INCLUDES = -I"$(TOMMATHDIR)" -PRJ_DEFINES = /DMP_PREC=4 /Dinline=__inline /D_CRT_SECURE_NO_DEPRECATE /D_CRT_NONSTDC_NO_DEPRECATE /DMP_FIXED_CUTOFFS /DTCL_LOAD_FROM_MEMORY +PRJ_DEFINES = /DMP_PREC=4 /Dinline=__inline /D_CRT_SECURE_NO_DEPRECATE /D_CRT_NONSTDC_NO_DEPRECATE /DMP_FIXED_CUTOFFS + +!if $(TCL_LOAD_FROM_MEMORY) +PRJ_DEFINES = $(PRJ_DEFINES) /DTCL_LOAD_FROM_MEMORY +!endif !if $(STATIC_BUILD) PRJ_DEFINES = $(PRJ_DEFINES) /DTCL_WITH_INTERNAL_ZLIB @@ -903,6 +916,10 @@ $(TMP_DIR)\tclTestObj.obj: $(GENERICDIR)\tclTestObj.c $(TMP_DIR)\tclTestABSList.obj: $(GENERICDIR)\tclTestABSList.c $(cc32) $(appcflags) -Fo$@ $? +$(TMP_DIR)\tclWinLoad.obj: $(WIN_DIR)\tclWinLoad.c + $(cc32) $(pkgcflags) /DUNICODE /D_UNICODE \ + -Fo$@ $? + $(TMP_DIR)\tclWinTest.obj: $(WIN_DIR)\tclWinTest.c $(CCAPPCMD) $? diff --git a/win/tclWinLoad.c b/win/tclWinLoad.c index ff268273068..0a422858e76 100644 --- a/win/tclWinLoad.c +++ b/win/tclWinLoad.c @@ -13,10 +13,6 @@ #include "tclWinInt.h" #ifdef TCL_LOAD_FROM_MEMORY -#undef UNICODE -#undef _UNICODE -#define UNICODE -#define _UNICODE #include "MemoryModule.c" #endif From f9e1f7c80c121360a4fce8dab3abda057acdc326 Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Wed, 27 Nov 2024 11:24:18 +0000 Subject: [PATCH 19/33] Build with MemoryModule on Github --- .github/workflows/onefiledist.yml | 2 +- .github/workflows/win-build.yml | 26 +++++++++++++------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/onefiledist.yml b/.github/workflows/onefiledist.yml index 542221d952e..521e4ed9fb2 100644 --- a/.github/workflows/onefiledist.yml +++ b/.github/workflows/onefiledist.yml @@ -125,7 +125,7 @@ jobs: timeout-minutes: 10 env: CC: gcc - CFGOPT: --disable-symbols --disable-shared + CFGOPT: --enable-memorymodule --disable-symbols --disable-shared steps: - name: Install MSYS2 uses: msys2/setup-msys2@v2 diff --git a/.github/workflows/win-build.yml b/.github/workflows/win-build.yml index aff50085490..5e9475af95e 100644 --- a/.github/workflows/win-build.yml +++ b/.github/workflows/win-build.yml @@ -21,12 +21,12 @@ jobs: strategy: matrix: config: - - "" - - "CHECKS=nodep" - - "OPTS=static" - - "OPTS=noembed" - - "OPTS=symbols" - - "OPTS=symbols STATS=compdbg,memdbg" + - "OPTS=memorymodule" + - "CHECKS=nodep OPTS=memorymodule" + - "OPTS=static,memorymodule" + - "OPTS=noembed,memorymodule" + - "OPTS=symbols,memorymodule" + - "OPTS=symbols,memorymodule STATS=compdbg,memdbg" # Using powershell means we need to explicitly stop on failure steps: - name: Checkout @@ -65,13 +65,13 @@ jobs: strategy: matrix: config: - - "" - - "CFLAGS=-DTCL_NO_DEPRECATED=1" - - "--disable-shared" - - "--disable-zipfs" - - "--enable-symbols" - - "--enable-symbols=mem" - - "--enable-symbols=all" + - "--enable-memorymodule" + - "--enable-memorymodule CFLAGS=-DTCL_NO_DEPRECATED=1" + - "--enable-memorymodule --disable-shared" + - "--enable-memorymodule --disable-zipfs" + - "--enable-memorymodule --enable-symbols" + - "--enable-memorymodule --enable-symbols=mem" + - "--enable-memorymodule --enable-symbols=all" # Using powershell means we need to explicitly stop on failure steps: - name: Install MSYS2 From 0cdf92372d00abcf44ce7d2b0180be388a8f3849 Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Wed, 27 Nov 2024 11:27:29 +0000 Subject: [PATCH 20/33] Fix [https://github.com/fancycode/MemoryModule/pull/91|Added x64 exception handling] --- win/MemoryModule.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/win/MemoryModule.c b/win/MemoryModule.c index 8c97caeeb1f..c5aae3cbad7 100644 --- a/win/MemoryModule.c +++ b/win/MemoryModule.c @@ -441,6 +441,17 @@ PerformBaseRelocation(PMEMORYMODULE module, ptrdiff_t delta) return TRUE; } +#ifdef _WIN64 +static BOOL +RegisterExceptionHandling(PMEMORYMODULE module) +{ + PIMAGE_DATA_DIRECTORY pDir = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_EXCEPTION); + PRUNTIME_FUNCTION pEntry = (PRUNTIME_FUNCTION)(module->codeBase + pDir->VirtualAddress); + UINT count = (pDir->Size / sizeof(IMAGE_RUNTIME_FUNCTION_ENTRY)) - 1; + return RtlAddFunctionTable(pEntry, count, (DWORD64)module->codeBase); +} +#endif + static BOOL BuildImportTable(PMEMORYMODULE module) { @@ -729,6 +740,12 @@ HMEMORYMODULE MemoryLoadLibraryEx(const void *data, size_t size, goto error; } +#ifdef _WIN64 + if (!RegisterExceptionHandling(result)) { + goto error; + } +#endif + // mark memory pages depending on section headers and release // sections that are marked as "discardable" if (!FinalizeSections(result)) { From 2ffc41b35193437fb54665bbbfdfba198bda96eb Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Thu, 28 Nov 2024 13:49:41 +0000 Subject: [PATCH 21/33] Make a start with TLS (gcc) and GetModuleName[A|W]. WIP. Make a start with test-cases --- win/configure | 3 +- win/configure.ac | 2 +- win/dltest/Makefile.in | 44 +++++++ win/dltest/memorymoduletest.c | 216 ++++++++++++++++++++++++++++++++++ win/dltest/pkgIndex.tcl | 12 ++ win/tclWinLoad.c | 55 ++++++++- 6 files changed, 329 insertions(+), 3 deletions(-) create mode 100644 win/dltest/Makefile.in create mode 100644 win/dltest/memorymoduletest.c create mode 100644 win/dltest/pkgIndex.tcl diff --git a/win/configure b/win/configure index f1ff71cefc3..052b2ee69be 100755 --- a/win/configure +++ b/win/configure @@ -6029,7 +6029,7 @@ TCL_WIN_VERSION="$TCL_VERSION.$TCL_RELEASE_LEVEL.`echo $TCL_PATCH_LEVEL | tr -d -ac_config_files="$ac_config_files Makefile tclConfig.sh tclsh.exe.manifest" +ac_config_files="$ac_config_files Makefile dltest/Makefile tclConfig.sh tclsh.exe.manifest" cat >confcache <<\_ACEOF # This file is a shell script that caches the results of configure @@ -6733,6 +6733,7 @@ for ac_config_target in $ac_config_targets do case $ac_config_target in "Makefile") CONFIG_FILES="$CONFIG_FILES Makefile" ;; + "dltest/Makefile") CONFIG_FILES="$CONFIG_FILES dltest/Makefile" ;; "tclConfig.sh") CONFIG_FILES="$CONFIG_FILES tclConfig.sh" ;; "tclsh.exe.manifest") CONFIG_FILES="$CONFIG_FILES tclsh.exe.manifest" ;; diff --git a/win/configure.ac b/win/configure.ac index 42c3b53cdb9..35dcd8ef3ea 100644 --- a/win/configure.ac +++ b/win/configure.ac @@ -480,7 +480,7 @@ AC_SUBST(RC_DEFINE) AC_SUBST(RC_DEFINES) AC_SUBST(RES) -AC_CONFIG_FILES([Makefile tclConfig.sh tclsh.exe.manifest]) +AC_CONFIG_FILES([Makefile dltest/Makefile tclConfig.sh tclsh.exe.manifest]) AC_OUTPUT dnl Local Variables: diff --git a/win/dltest/Makefile.in b/win/dltest/Makefile.in new file mode 100644 index 00000000000..5eeb007f6ee --- /dev/null +++ b/win/dltest/Makefile.in @@ -0,0 +1,44 @@ +# This Makefile is used to create several test cases for Tcl's load +# command. It also illustrates how to take advantage of configuration +# exported by Tcl to set up Makefiles for shared libraries. + +CC = @CC@ +LIBS = @TCL_BUILD_STUB_LIB_SPEC@ +AC_FLAGS = @DEFS@ +SHLIB_LD = @SHLIB_LD@ +SHLIB_CFLAGS = @SHLIB_CFLAGS@ +SHLIB_LD_LIBS = @SHLIB_LD_LIBS@ +SHLIB_SUFFIX = @SHLIB_SUFFIX@ +DLTEST_LD = @DLTEST_LD@ +SRC_DIR = @TCL_SRC_DIR@/win/dltest +BUILD_DIR = @builddir@ +TCL_VERSION= @TCL_VERSION@ + +CFLAGS_DEBUG = @CFLAGS_DEBUG@ +CFLAGS_OPTIMIZE = @CFLAGS_OPTIMIZE@ +CFLAGS = @CFLAGS_DEFAULT@ @CFLAGS@ -DTCL_NO_DEPRECATED=1 -Wall -Wextra -Wc++-compat -Wconversion -Werror +LDFLAGS_DEBUG = @LDFLAGS_DEBUG@ +LDFLAGS_OPTIMIZE = @LDFLAGS_OPTIMIZE@ +LDFLAGS = @LDFLAGS_DEFAULT@ @LDFLAGS@ -static-libgcc + +CC_SWITCHES = $(CFLAGS) -I${SRC_DIR}/../../generic \ + ${SHLIB_CFLAGS} -DUSE_TCL_STUBS ${AC_FLAGS} + +all: tcl9memorymoduletest${SHLIB_SUFFIX} + @touch ../dltest.marker + +memorymoduletest.o: $(SRC_DIR)/memorymoduletest.c + $(CC) -c $(CC_SWITCHES) $(SRC_DIR)/memorymoduletest.c + +tcl9memorymoduletest${SHLIB_SUFFIX}: memorymoduletest.o + ${SHLIB_LD} -o $@ memorymoduletest.o ${SHLIB_LD_LIBS} + +clean: + rm -f embtest *.o lib.exp ../dltest.marker + @if test "$(SHLIB_SUFFIX)" != ""; then \ + echo "rm -f *${SHLIB_SUFFIX}" ; \ + rm -f *${SHLIB_SUFFIX} ; \ + fi + +distclean: clean + rm -f Makefile diff --git a/win/dltest/memorymoduletest.c b/win/dltest/memorymoduletest.c new file mode 100644 index 00000000000..ca44d02dfb0 --- /dev/null +++ b/win/dltest/memorymoduletest.c @@ -0,0 +1,216 @@ +/* + * memorymoduletest.c -- + * + * This file contains a simple Tcl package "memorymoduletest" that is intended for + * testing the Tcl dynamic loading facilities using MemoryModule. + * + * Copyright © 2024 Jan Nijtmans + * + * See the file "license.terms" for information on usage and redistribution of + * this file, and for a DISCLAIMER OF ALL WARRANTIES. + */ + +#undef STATIC_BUILD +#include +#include +#include "tcl.h" + +static HMODULE hModule = NULL; + +static bool threadAttachCalled = false; + +#if 1 /* disabled, because it doesn't work */ +#ifdef _MSC_VER +__declspec(thread) +#elif defined __GNUC__ +__thread +#elif __cplusplus +thread_local +#endif +#endif +int threadVar = 0; + +BOOL APIENTRY +DllMain( + HINSTANCE hInst, /* Library instance handle. */ + DWORD reason, /* Reason this function is being called. */ + LPVOID unused) +{ + (void)reason; + (void)unused; + + switch (reason) { + case DLL_PROCESS_ATTACH: + hModule = hInst; + break; + case DLL_THREAD_ATTACH: + threadAttachCalled = true; + break; + default: + break; + } + return TRUE; +} + + +/* + *---------------------------------------------------------------------- + * + * Mm_ModuleFileNameACmd -- + * + * This procedure is invoked to process the "GetModuleFileNameA" Tcl command. + * It calls the Windows GetModuleFileNameA(), and gives back the result. + * + * Results: + * A standard Tcl result. + * + * Side effects: + * See the user documentation. + * + *---------------------------------------------------------------------- + */ + +static int +Mmt_ModuleFileNameACmd( + void *dummy, /* Not used. */ + Tcl_Interp *interp, /* Current interpreter. */ + int objc, /* Number of arguments. */ + Tcl_Obj *const objv[]) /* Argument objects. */ +{ + char buffer[MAX_PATH]; + (void)dummy; + + if (objc != 1) { + Tcl_WrongNumArgs(interp, 1, objv, ""); + return TCL_ERROR; + } + if (GetModuleFileNameA(hModule, buffer, MAX_PATH) == 0) { + Tcl_WinConvertError(GetLastError()); + Tcl_SetObjResult(interp, Tcl_ObjPrintf("could not determine ModuleFileName: %s", + Tcl_PosixError(interp))); + return TCL_ERROR; + } + Tcl_SetObjResult(interp, Tcl_NewStringObj(buffer, -1)); + return TCL_OK; +} + +/* + *---------------------------------------------------------------------- + * + * Mm_ModuleFileNameWCmd -- + * + * This procedure is invoked to process the "GetModuleFileNameW" Tcl command. + * It calls the Windows GetModuleFileNameW(), and gives back the result. + * + * Results: + * A standard Tcl result. + * + * Side effects: + * See the user documentation. + * + *---------------------------------------------------------------------- + */ + +static int +Mmt_ModuleFileNameWCmd( + void *dummy, /* Not used. */ + Tcl_Interp *interp, /* Current interpreter. */ + int objc, /* Number of arguments. */ + Tcl_Obj *const objv[]) /* Argument strings. */ +{ + WCHAR buffer[MAX_PATH]; + Tcl_DString ds; + (void)dummy; + + if (objc != 1) { + Tcl_WrongNumArgs(interp, 1, objv, ""); + return TCL_ERROR; + } + if (GetModuleFileNameW(hModule, buffer, MAX_PATH) == 0) { + Tcl_WinConvertError(GetLastError()); + Tcl_SetObjResult(interp, Tcl_ObjPrintf("could not determine ModuleFileName: %s", + Tcl_PosixError(interp))); + return TCL_ERROR; + } + Tcl_DStringInit(&ds); + Tcl_WCharToUtfDString(buffer, -1, &ds); + Tcl_SetObjResult(interp, Tcl_DStringToObj(&ds)); + return TCL_OK; +} + +static int +Mmt_ThreadAttachCalled( + void *dummy, /* Not used. */ + Tcl_Interp *interp, /* Current interpreter. */ + int objc, /* Number of arguments. */ + Tcl_Obj *const objv[]) /* Argument strings. */ +{ + (void)dummy; + + if (objc != 1) { + Tcl_WrongNumArgs(interp, 1, objv, ""); + return TCL_ERROR; + } + Tcl_SetObjResult(interp, Tcl_NewBooleanObj(threadAttachCalled)); + return TCL_OK; +} + +static int +Mmt_ThreadVar( + void *dummy, /* Not used. */ + Tcl_Interp *interp, /* Current interpreter. */ + int objc, /* Number of arguments. */ + Tcl_Obj *const objv[]) /* Argument strings. */ +{ + (void)dummy; + + if (objc > 2) { + Tcl_WrongNumArgs(interp, 1, objv, ""); + return TCL_ERROR; + } + if (objc == 2) { + if (Tcl_GetIntFromObj(interp, objv[1], &threadVar) != TCL_OK) { + return TCL_ERROR; + } + } + Tcl_SetObjResult(interp, Tcl_NewIntObj(threadVar)); + return TCL_OK; +} + +/* + *---------------------------------------------------------------------- + * + * Memorymoduletest_Init -- + * + * This is a package initialization procedure, which is called by Tcl + * when this package is to be added to an interpreter. + * + * Results: + * None. + * + * Side effects: + * None. + * + *---------------------------------------------------------------------- + */ + +DLLEXPORT int +Memorymoduletest_Init( + Tcl_Interp *interp) /* Interpreter in which the package is to be + * made available. */ +{ + int code; + + if (Tcl_InitStubs(interp, "8.6-", 0) == NULL) { + return TCL_ERROR; + } + code = Tcl_PkgProvide(interp, "memorymoduletest", "1.0.0"); + if (code != TCL_OK) { + return code; + } + Tcl_CreateObjCommand(interp, "GetModuleFileNameA", Mmt_ModuleFileNameACmd, NULL, NULL); + Tcl_CreateObjCommand(interp, "GetModuleFileNameW", Mmt_ModuleFileNameWCmd, NULL, NULL); + Tcl_CreateObjCommand(interp, "ThreadAttachCalled", Mmt_ThreadAttachCalled, NULL, NULL); + Tcl_CreateObjCommand(interp, "ThreadVar", Mmt_ThreadVar, NULL, NULL); + return TCL_OK; +} diff --git a/win/dltest/pkgIndex.tcl b/win/dltest/pkgIndex.tcl new file mode 100644 index 00000000000..010657c8c33 --- /dev/null +++ b/win/dltest/pkgIndex.tcl @@ -0,0 +1,12 @@ +# Tcl package index file, version 1.1 +# This file is generated by the "pkg_mkIndex -direct" command +# and sourced either when an application starts up or +# by a "package unknown" script. It invokes the +# "package ifneeded" command to set up package-related +# information so that packages will be loaded automatically +# in response to "package require" commands. When this +# script is sourced, the variable $dir must contain the +# full path name of this file's directory. + +if {![package vsatisfies [package provide Tcl] 9.0]} {return} +package ifneeded memorymoduletest 1.0.0 [list load [file join $dir tcl9memorymoduletest.dll]] diff --git a/win/tclWinLoad.c b/win/tclWinLoad.c index 0a422858e76..7c0f4766734 100644 --- a/win/tclWinLoad.c +++ b/win/tclWinLoad.c @@ -400,6 +400,44 @@ InitDLLDirectoryName(void) #ifdef TCL_LOAD_FROM_MEMORY +struct __emutls_object +{ + size_t size; + size_t align; + void *ptr; + void *templ; +}; +struct __emutls_array +{ + void *skip_destructor_rounds; + void *size; + void **data[]; +}; + +static char tls_storage[256] = {0}; + +static void *fake_emutls_get_address(struct __emutls_object *obj) { + (void)obj; + // TODO to be implemented. For now just point to some static storage. + return tls_storage; +} + +static int fake_GetModuleFileNameW(void *module, WCHAR *path) { + (void)module; + static const WCHAR wpath[] = L"TODO.dll"; + // TODO to be implemented. For now just point to some static storage. + wcscpy(path, wpath); + return wcslen(wpath); +} + +static int fake_GetModuleFileNameA(void *module, char *path) { + (void)module; + static const char apath[] = "TODO.dll"; + // TODO to be implemented. For now just point to some static storage. + strcpy(path, apath); + return strlen(apath); +} + MODULE_SCOPE void * TclpLoadMemoryGetBuffer( size_t size) @@ -434,6 +472,19 @@ FindMemSymbol( return res; } +static FARPROC +FakeDefaultGetProcAddress(HCUSTOMMODULE module, LPCSTR name, void *userdata) +{ + if (!strcmp(name, "__emutls_get_address")) { + return (FARPROC)(void *)fake_emutls_get_address; + } else if (!strcmp(name, "GetModuleFileNameW")) { + return (FARPROC)(void *)fake_GetModuleFileNameW; + } else if (!strcmp(name, "GetModuleFileNameA")) { + return (FARPROC)(void *)fake_GetModuleFileNameA; + } + return MemoryDefaultGetProcAddress(module, name, userdata); +} + MODULE_SCOPE int TclpLoadMemory( void *data, /* Buffer containing the desired code @@ -458,7 +509,9 @@ TclpLoadMemory( Tcl_Free(data); return TCL_ERROR; } - hInstance = MemoryLoadLibrary(data, size); + hInstance = MemoryLoadLibraryEx(data, size, MemoryDefaultAlloc, MemoryDefaultFree, + MemoryDefaultLoadLibrary, FakeDefaultGetProcAddress, MemoryDefaultFreeLibrary, NULL); + if (hInstance == NULL) { return TCL_ERROR; } From e85ecf30971eb93e8095e9b5229bfedb031d207b Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Thu, 28 Nov 2024 15:40:26 +0000 Subject: [PATCH 22/33] Add makefile.vc --- win/dltest/makefile.vc | 44 +++++++++++++++++++++++++++++++++++ win/dltest/memorymoduletest.c | 5 ++-- 2 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 win/dltest/makefile.vc diff --git a/win/dltest/makefile.vc b/win/dltest/makefile.vc new file mode 100644 index 00000000000..b1aaf9ba4fe --- /dev/null +++ b/win/dltest/makefile.vc @@ -0,0 +1,44 @@ +TOP = . + +# optional build flags +LOC = + +# variables +SHAREDLIB = tcl9memorymoduletest.dll +IMPLIB = memorymoduletest.lib + +CC = cl +AS = ml +LD = link +CFLAGS = -nologo -MD -W3 -O2 -Oy- -Zi $(LOC) -DUSE_TCL_STUBS -I../../generic +WFLAGS = -D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE +LDFLAGS = -nologo -debug -incremental:no -opt:ref + +OBJS = memorymoduletest.obj + +# targets +all: $(SHAREDLIB) $(IMPLIB) + +$(IMPLIB): $(SHAREDLIB) + +$(SHAREDLIB): $(OBJS) + $(LD) $(LDFLAGS) -dll -implib:$(IMPLIB) tclstub.lib \ + -out:$@ $(OBJS) + +{$(TOP)}.c.obj: + $(CC) -c $(WFLAGS) $(CFLAGS) $< + +memorymoduletest.obj: $(TOP)/memorymoduletest.c + +# cleanup +clean: + -del $(STATICLIB) + -del $(SHAREDLIB) + -del $(IMPLIB) + -del *.obj + -del *.res + -del *.exp + -del *.exe + -del *.pdb + -del *.manifest + -del foo.gz diff --git a/win/dltest/memorymoduletest.c b/win/dltest/memorymoduletest.c index ca44d02dfb0..943f3722fc0 100644 --- a/win/dltest/memorymoduletest.c +++ b/win/dltest/memorymoduletest.c @@ -19,7 +19,6 @@ static HMODULE hModule = NULL; static bool threadAttachCalled = false; -#if 1 /* disabled, because it doesn't work */ #ifdef _MSC_VER __declspec(thread) #elif defined __GNUC__ @@ -27,7 +26,6 @@ __thread #elif __cplusplus thread_local #endif -#endif int threadVar = 0; BOOL APIENTRY @@ -201,7 +199,8 @@ Memorymoduletest_Init( { int code; - if (Tcl_InitStubs(interp, "8.6-", 0) == NULL) { + if (Tcl_InitStubs(interp, "8.7-", 0) == NULL) { + /* Tcl 8.6 doesn't have Tcl_DStringToObj() */ return TCL_ERROR; } code = Tcl_PkgProvide(interp, "memorymoduletest", "1.0.0"); From 0fb12b53b18b916650ef83e3cbd93811141d1a28 Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Fri, 29 Nov 2024 08:20:30 +0000 Subject: [PATCH 23/33] Add more testcases. Store full path in loadHandle --- generic/tclIOUtil.c | 2 +- generic/tclInt.h | 5 +-- generic/tclLoadNone.c | 1 + tests/memorymodule.test | 48 ++++++++++++++++++++++++++ unix/tclLoadDyld.c | 1 + win/dltest/Makefile.in | 13 +++++-- win/tclWinLoad.c | 76 +++++++++++++++++++---------------------- 7 files changed, 100 insertions(+), 46 deletions(-) create mode 100644 tests/memorymodule.test diff --git a/generic/tclIOUtil.c b/generic/tclIOUtil.c index bd39c4a567e..77bc1c0d699 100644 --- a/generic/tclIOUtil.c +++ b/generic/tclIOUtil.c @@ -3255,7 +3255,7 @@ Tcl_LoadFile( } ret = Tcl_Read(data, (char *)buffer, size); Tcl_CloseEx(interp, data, 0); - ret = TclpLoadMemory(buffer, size, ret, handlePtr, + ret = TclpLoadMemory(buffer, size, ret, TclGetString(pathPtr), handlePtr, &unloadProcPtr, flags); if (ret == TCL_OK && *handlePtr != NULL) { goto resolveSymbols; diff --git a/generic/tclInt.h b/generic/tclInt.h index 0a6899e6ca8..48caab72673 100644 --- a/generic/tclInt.h +++ b/generic/tclInt.h @@ -3263,6 +3263,7 @@ struct Tcl_LoadHandle_ { * loaded module */ Tcl_FSUnloadFileProc* unloadFileProcPtr; /* Procedure that unloads a loaded module */ + char name[TCLFLEXARRAY]; }; /* Flags for conversion of doubles to digit strings */ @@ -3669,8 +3670,8 @@ MODULE_SCOPE int TclpDlopen(Tcl_Interp *interp, Tcl_Obj *pathPtr, MODULE_SCOPE int TclpUtime(Tcl_Obj *pathPtr, struct utimbuf *tval); #ifdef TCL_LOAD_FROM_MEMORY MODULE_SCOPE void * TclpLoadMemoryGetBuffer(size_t size); -MODULE_SCOPE int TclpLoadMemory(void *buffer, - size_t size, Tcl_Size codeSize, Tcl_LoadHandle *loadHandle, +MODULE_SCOPE int TclpLoadMemory(void *buffer, size_t size, + Tcl_Size codeSize, const char *path, Tcl_LoadHandle *loadHandle, Tcl_FSUnloadFileProc **unloadProcPtr, int flags); #endif MODULE_SCOPE void TclInitThreadStorage(void); diff --git a/generic/tclLoadNone.c b/generic/tclLoadNone.c index 273ebc77af7..8bb43a398c4 100644 --- a/generic/tclLoadNone.c +++ b/generic/tclLoadNone.c @@ -73,6 +73,7 @@ TclpLoadMemory( TCL_UNUSED(void *), TCL_UNUSED(size_t), TCL_UNUSED(Tcl_Size), + TCL_UNUSED(const char *), TCL_UNUSED(Tcl_LoadHandle *), TCL_UNUSED(Tcl_FSUnloadFileProc **), TCL_UNUSED(int)) diff --git a/tests/memorymodule.test b/tests/memorymodule.test new file mode 100644 index 00000000000..4751aea79b5 --- /dev/null +++ b/tests/memorymodule.test @@ -0,0 +1,48 @@ +if {"::tcltest" ni [namespace children]} { + package require tcltest 2.5 + namespace import -force ::tcltest::* +} + +testConstraint thread [expr {0 == [catch {package require Thread 2.7-}]}] +testConstraint memorymoduletest [expr {0 == [catch {zipfs mount dltest/memorymoduletest.zip [file join [zipfs root] memorymoduletest]}]}] + +lappend auto_path [file join [zipfs root] memorymoduletest] + +test memorymodule-1.0 {info loaded} memorymoduletest { + package require memorymoduletest + info loaded {} Memorymoduletest +} [file join [zipfs root] memorymoduletest/tcl9memorymoduletest.dll] + +test memorymodule-1.1 {GetModuleFileNameA (WIP)} memorymoduletest { + package require memorymoduletest + GetModuleFileNameA +} TODOA.dll + +test memorymodule-1.2 {GetModuleFileNameW (WIP)} memorymoduletest { + package require memorymoduletest + GetModuleFileNameW +} TODOW.dll + +test memorymodule-2.0 {LTS} -constraints {memorymoduletest thread} -body { + package require Thread + package require memorymoduletest + set t1 [thread::create] + ThreadVar 15; # Set ThreadVar to 15 in the main thread + thread::preserve $t1 + thread::send $t1 { + lappend auto_path [file join [zipfs root] memorymoduletest] + package require memorymoduletest + # set ThreadVar to 16 in the subthread + ThreadVar 16 + return [ThreadVar] + } result + thread::release $t1 + # ThreadVar in main thread should be unchanged (15) + list [ThreadVar] $result +} -result {15 16} + + +# cleanup +::tcltest::cleanupTests +return + diff --git a/unix/tclLoadDyld.c b/unix/tclLoadDyld.c index e2b31a638d5..7b6dbc4c195 100644 --- a/unix/tclLoadDyld.c +++ b/unix/tclLoadDyld.c @@ -546,6 +546,7 @@ TclpLoadMemory( Tcl_Size codeSize, /* Size of code data read into buffer or -1 if * an error occurred and the buffer should * just be freed. */ + TCL_UNUSED(const char *), Tcl_LoadHandle *loadHandle, /* Filled with token for dynamically loaded * file which will be passed back to * (*unloadProcPtr)() to unload the file. */ diff --git a/win/dltest/Makefile.in b/win/dltest/Makefile.in index 5eeb007f6ee..75a223c8e3a 100644 --- a/win/dltest/Makefile.in +++ b/win/dltest/Makefile.in @@ -12,7 +12,8 @@ SHLIB_SUFFIX = @SHLIB_SUFFIX@ DLTEST_LD = @DLTEST_LD@ SRC_DIR = @TCL_SRC_DIR@/win/dltest BUILD_DIR = @builddir@ -TCL_VERSION= @TCL_VERSION@ +NATIVE_ZIP = @ZIP_PROG@ +TCL_VERSION = @TCL_VERSION@ CFLAGS_DEBUG = @CFLAGS_DEBUG@ CFLAGS_OPTIMIZE = @CFLAGS_OPTIMIZE@ @@ -24,8 +25,14 @@ LDFLAGS = @LDFLAGS_DEFAULT@ @LDFLAGS@ -static-libgcc CC_SWITCHES = $(CFLAGS) -I${SRC_DIR}/../../generic \ ${SHLIB_CFLAGS} -DUSE_TCL_STUBS ${AC_FLAGS} -all: tcl9memorymoduletest${SHLIB_SUFFIX} - @touch ../dltest.marker +all: memorymoduletest.zip + +memorymoduletest.zip: tcl9memorymoduletest${SHLIB_SUFFIX} + (zip=`(realpath '${NATIVE_ZIP}' || readlink -m '${NATIVE_ZIP}') 2>/dev/null || \ + (echo '${NATIVE_ZIP}' | sed "s?^\./?$$(pwd)/?")`; \ + $$zip memorymoduletest.zip pkgIndex.tcl *.dll >/dev/null && \ + echo "memorymoduletest.zip successful created with $$zip" \ + ) memorymoduletest.o: $(SRC_DIR)/memorymoduletest.c $(CC) -c $(CC_SWITCHES) $(SRC_DIR)/memorymoduletest.c diff --git a/win/tclWinLoad.c b/win/tclWinLoad.c index 7c0f4766734..65faae576b5 100644 --- a/win/tclWinLoad.c +++ b/win/tclWinLoad.c @@ -400,42 +400,21 @@ InitDLLDirectoryName(void) #ifdef TCL_LOAD_FROM_MEMORY -struct __emutls_object -{ - size_t size; - size_t align; - void *ptr; - void *templ; -}; -struct __emutls_array -{ - void *skip_destructor_rounds; - void *size; - void **data[]; -}; - -static char tls_storage[256] = {0}; - -static void *fake_emutls_get_address(struct __emutls_object *obj) { - (void)obj; - // TODO to be implemented. For now just point to some static storage. - return tls_storage; -} +static HMODULE kernel32 = NULL; +static Tcl_HashTable vfsPathTable; static int fake_GetModuleFileNameW(void *module, WCHAR *path) { - (void)module; - static const WCHAR wpath[] = L"TODO.dll"; + (void)module; // TODO to be implemented. For now just point to some static storage. - wcscpy(path, wpath); - return wcslen(wpath); + wcscpy(path, L"TODOW.dll"); + return wcslen(path); } static int fake_GetModuleFileNameA(void *module, char *path) { - (void)module; - static const char apath[] = "TODO.dll"; + (void)module; // TODO to be implemented. For now just point to some static storage. - strcpy(path, apath); - return strlen(apath); + strcpy(path, "TODOA.dll"); + return strlen(path); } MODULE_SCOPE void * @@ -475,12 +454,22 @@ FindMemSymbol( static FARPROC FakeDefaultGetProcAddress(HCUSTOMMODULE module, LPCSTR name, void *userdata) { - if (!strcmp(name, "__emutls_get_address")) { - return (FARPROC)(void *)fake_emutls_get_address; - } else if (!strcmp(name, "GetModuleFileNameW")) { - return (FARPROC)(void *)fake_GetModuleFileNameW; - } else if (!strcmp(name, "GetModuleFileNameA")) { - return (FARPROC)(void *)fake_GetModuleFileNameA; + Tcl_LoadHandle loadHandle = (Tcl_LoadHandle)userdata; + Tcl_HashEntry *entry; + int isNew; + + if (kernel32 == NULL) { + kernel32 = GetModuleHandleW(L"KERNEL32"); + Tcl_InitHashTable(&vfsPathTable, TCL_ONE_WORD_KEYS); + } + if ((module == kernel32) && !strcmp(name, "GetModuleFileNameW")) { + entry = Tcl_CreateHashEntry(&vfsPathTable, module, &isNew); + Tcl_SetHashValue(entry, loadHandle->name); + return (FARPROC)(void *)fake_GetModuleFileNameW; + } else if ((module == kernel32) && !strcmp(name, "GetModuleFileNameA")) { + entry = Tcl_CreateHashEntry(&vfsPathTable, module, &isNew); + Tcl_SetHashValue(entry, loadHandle->name); + return (FARPROC)(void *)fake_GetModuleFileNameA; } return MemoryDefaultGetProcAddress(module, name, userdata); } @@ -493,6 +482,7 @@ TclpLoadMemory( Tcl_Size codeSize, /* Size of code data read into buffer or TCL_INDEX_NONE * if an error occurred and buffer should be * free'd. */ + const char *path, /* path in VFS, or NULL if the path is unknown */ Tcl_LoadHandle *loadHandle, /* Filled with token for dynamically loaded * file which will be passed back to * (*unloadProcPtr)() to unload the file. */ @@ -504,21 +494,27 @@ TclpLoadMemory( { Tcl_LoadHandle handlePtr; void *hInstance; + size_t handleSize = offsetof(struct Tcl_LoadHandle_, name) + (path ? strlen(path) : 1); if (codeSize < 1) { Tcl_Free(data); return TCL_ERROR; } + handlePtr = (Tcl_LoadHandle)Tcl_Alloc(handleSize); + handlePtr->findSymbolProcPtr = &FindMemSymbol; + handlePtr->unloadFileProcPtr = &UnloadMemory; + if (path) { + strcpy(handlePtr->name, path); + } else { + handlePtr->name[0] = 0; + } hInstance = MemoryLoadLibraryEx(data, size, MemoryDefaultAlloc, MemoryDefaultFree, - MemoryDefaultLoadLibrary, FakeDefaultGetProcAddress, MemoryDefaultFreeLibrary, NULL); - + MemoryDefaultLoadLibrary, FakeDefaultGetProcAddress, MemoryDefaultFreeLibrary, handlePtr); if (hInstance == NULL) { + Tcl_Free(handlePtr); return TCL_ERROR; } - handlePtr = (Tcl_LoadHandle)Tcl_Alloc(sizeof(struct Tcl_LoadHandle_)); handlePtr->clientData = hInstance; - handlePtr->findSymbolProcPtr = &FindMemSymbol; - handlePtr->unloadFileProcPtr = &UnloadMemory; *loadHandle = handlePtr; *unloadProcPtr = &UnloadMemory; return TCL_OK; From 258845894fd9a9850eb18ae7c478d728d03a378a Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Fri, 29 Nov 2024 21:05:56 +0000 Subject: [PATCH 24/33] New function MemoryGetCodeBase(). We need it. --- win/MemoryModule.c | 5 +++++ win/MemoryModule.h | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/win/MemoryModule.c b/win/MemoryModule.c index 67db5badc4c..43e83475086 100644 --- a/win/MemoryModule.c +++ b/win/MemoryModule.c @@ -1116,6 +1116,11 @@ LPVOID MemoryLoadResource(HMEMORYMODULE module, HMEMORYRSRC resource) return codeBase + entry->OffsetToData; } +HMODULE MemoryGetCodeBase(HMEMORYMODULE module) +{ + return (HMODULE)((PMEMORYMODULE)module)->codeBase; +} + int MemoryLoadString(HMEMORYMODULE module, UINT id, LPTSTR buffer, int maxsize) { diff --git a/win/MemoryModule.h b/win/MemoryModule.h index a728f6b1416..55e96c3d382 100644 --- a/win/MemoryModule.h +++ b/win/MemoryModule.h @@ -111,6 +111,11 @@ DWORD MemorySizeofResource(HMEMORYMODULE, HMEMORYRSRC); */ LPVOID MemoryLoadResource(HMEMORYMODULE, HMEMORYRSRC); +/** + * Get a pointer to the hInstance. + */ +HMODULE MemoryGetCodeBase(HMEMORYMODULE); + /** * Load a string resource. */ From e25a1e7c7ead42f5d74a09a495f8d18990c9496a Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Fri, 29 Nov 2024 21:08:12 +0000 Subject: [PATCH 25/33] Now with a working GetModuleFileName[WA] for memory-loaded modules. --- tests/memorymodule.test | 4 ++-- win/tclWinLoad.c | 48 ++++++++++++++++++++++++----------------- 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/tests/memorymodule.test b/tests/memorymodule.test index 4751aea79b5..5f2195e4342 100644 --- a/tests/memorymodule.test +++ b/tests/memorymodule.test @@ -16,12 +16,12 @@ test memorymodule-1.0 {info loaded} memorymoduletest { test memorymodule-1.1 {GetModuleFileNameA (WIP)} memorymoduletest { package require memorymoduletest GetModuleFileNameA -} TODOA.dll +} //zipfs:/memorymoduletest/tcl9memorymoduletest.dll test memorymodule-1.2 {GetModuleFileNameW (WIP)} memorymoduletest { package require memorymoduletest GetModuleFileNameW -} TODOW.dll +} //zipfs:/memorymoduletest/tcl9memorymoduletest.dll test memorymodule-2.0 {LTS} -constraints {memorymoduletest thread} -body { package require Thread diff --git a/win/tclWinLoad.c b/win/tclWinLoad.c index 65faae576b5..536a5b56c2f 100644 --- a/win/tclWinLoad.c +++ b/win/tclWinLoad.c @@ -403,18 +403,25 @@ InitDLLDirectoryName(void) static HMODULE kernel32 = NULL; static Tcl_HashTable vfsPathTable; -static int fake_GetModuleFileNameW(void *module, WCHAR *path) { - (void)module; - // TODO to be implemented. For now just point to some static storage. - wcscpy(path, L"TODOW.dll"); - return wcslen(path); +static int fake_GetModuleFileNameW(HMODULE module, WCHAR *path, WORD nSize) { + Tcl_MutexLock(&dllDirectoryNameMutex); + Tcl_HashEntry *entry = Tcl_FindHashEntry(&vfsPathTable, module); + Tcl_MutexUnlock(&dllDirectoryNameMutex); + if (entry == NULL) { + return GetModuleFileNameW(module, path, nSize); + } + return MultiByteToWideChar(CP_UTF8, 0, (const char *)Tcl_GetHashValue(entry), -1 , path, nSize); } -static int fake_GetModuleFileNameA(void *module, char *path) { - (void)module; - // TODO to be implemented. For now just point to some static storage. - strcpy(path, "TODOA.dll"); - return strlen(path); +static int fake_GetModuleFileNameA(HMODULE module, LPSTR lpFilename, WORD nSize) { + Tcl_MutexLock(&dllDirectoryNameMutex); + Tcl_HashEntry *entry = Tcl_FindHashEntry(&vfsPathTable, module); + Tcl_MutexUnlock(&dllDirectoryNameMutex); + if (entry == NULL) { + return GetModuleFileNameA(module, lpFilename, nSize); + } + strncpy(lpFilename, (const char *)Tcl_GetHashValue(entry), nSize); + return strlen(lpFilename); } MODULE_SCOPE void * @@ -454,21 +461,17 @@ FindMemSymbol( static FARPROC FakeDefaultGetProcAddress(HCUSTOMMODULE module, LPCSTR name, void *userdata) { - Tcl_LoadHandle loadHandle = (Tcl_LoadHandle)userdata; - Tcl_HashEntry *entry; - int isNew; - if (kernel32 == NULL) { - kernel32 = GetModuleHandleW(L"KERNEL32"); - Tcl_InitHashTable(&vfsPathTable, TCL_ONE_WORD_KEYS); + Tcl_MutexLock(&dllDirectoryNameMutex); + if (kernel32 == NULL) { + kernel32 = GetModuleHandleW(L"KERNEL32"); + Tcl_InitHashTable(&vfsPathTable, TCL_ONE_WORD_KEYS); + } + Tcl_MutexUnlock(&dllDirectoryNameMutex); } if ((module == kernel32) && !strcmp(name, "GetModuleFileNameW")) { - entry = Tcl_CreateHashEntry(&vfsPathTable, module, &isNew); - Tcl_SetHashValue(entry, loadHandle->name); return (FARPROC)(void *)fake_GetModuleFileNameW; } else if ((module == kernel32) && !strcmp(name, "GetModuleFileNameA")) { - entry = Tcl_CreateHashEntry(&vfsPathTable, module, &isNew); - Tcl_SetHashValue(entry, loadHandle->name); return (FARPROC)(void *)fake_GetModuleFileNameA; } return MemoryDefaultGetProcAddress(module, name, userdata); @@ -495,6 +498,7 @@ TclpLoadMemory( Tcl_LoadHandle handlePtr; void *hInstance; size_t handleSize = offsetof(struct Tcl_LoadHandle_, name) + (path ? strlen(path) : 1); + int isNew; if (codeSize < 1) { Tcl_Free(data); @@ -510,6 +514,10 @@ TclpLoadMemory( } hInstance = MemoryLoadLibraryEx(data, size, MemoryDefaultAlloc, MemoryDefaultFree, MemoryDefaultLoadLibrary, FakeDefaultGetProcAddress, MemoryDefaultFreeLibrary, handlePtr); + Tcl_MutexLock(&dllDirectoryNameMutex); + Tcl_HashEntry *entry = Tcl_CreateHashEntry(&vfsPathTable, MemoryGetCodeBase(hInstance), &isNew); + Tcl_SetHashValue(entry, handlePtr->name); + Tcl_MutexLock(&dllDirectoryNameMutex); if (hInstance == NULL) { Tcl_Free(handlePtr); return TCL_ERROR; From 00f5cb7f86f4c778b9fe5b076828a39a851bb1ba Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Fri, 29 Nov 2024 22:59:50 +0000 Subject: [PATCH 26/33] Build "memorymodule" tag into tcl::build-info --- generic/tclEvent.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/generic/tclEvent.c b/generic/tclEvent.c index 4dd10d86e4a..6c78a635aa9 100644 --- a/generic/tclEvent.c +++ b/generic/tclEvent.c @@ -1072,6 +1072,9 @@ static const struct { #ifdef TCL_MEM_DEBUG ".memdebug" #endif +#if defined(_WIN32) && defined(TCL_LOAD_FROM_MEMORY) + ".memorymodule-0004" +#endif #if defined(_MSC_VER) ".msvc-" STRINGIFY(_MSC_VER) #endif From 15328b6c479fb73eb31e89b6bda7d9f78134add7 Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Sun, 1 Dec 2024 16:56:52 +0000 Subject: [PATCH 27/33] Integrate dltest/memorymoduletest.zip building in win/Makefile --- win/Makefile.in | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/win/Makefile.in b/win/Makefile.in index 3f7044cd0d3..a913bad378b 100644 --- a/win/Makefile.in +++ b/win/Makefile.in @@ -1027,11 +1027,14 @@ install-private-headers: libraries test: test-tcl test-packages -test-tcl: tcltest +test-tcl: tcltest dltest/memorymoduletest.zip TCL_LIBRARY="$(LIBRARY_DIR)"; export TCL_LIBRARY; \ $(WINE) ./$(TCLSH) "$(ROOT_DIR_NATIVE)/tests/all.tcl" $(TESTFLAGS) \ -load "$(TEST_LOAD_FACILITIES)" +dltest/memorymoduletest.zip: + ( cd dltest; make; ) + # Useful target to launch a built tclsh with the proper path,... runtest: tcltest @TCL_LIBRARY="$(LIBRARY_DIR)"; export TCL_LIBRARY; \ From 4f57fde5c45549367008b791e1a2c79b39036c47 Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Mon, 2 Dec 2024 11:53:26 +0000 Subject: [PATCH 28/33] Fix compiler warning. Protect against long pathnames. --- win/tclWinLoad.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/win/tclWinLoad.c b/win/tclWinLoad.c index 536a5b56c2f..771b1a4b3cb 100644 --- a/win/tclWinLoad.c +++ b/win/tclWinLoad.c @@ -403,7 +403,7 @@ InitDLLDirectoryName(void) static HMODULE kernel32 = NULL; static Tcl_HashTable vfsPathTable; -static int fake_GetModuleFileNameW(HMODULE module, WCHAR *path, WORD nSize) { +static DWORD fake_GetModuleFileNameW(HMODULE module, WCHAR *path, WORD nSize) { Tcl_MutexLock(&dllDirectoryNameMutex); Tcl_HashEntry *entry = Tcl_FindHashEntry(&vfsPathTable, module); Tcl_MutexUnlock(&dllDirectoryNameMutex); @@ -413,7 +413,7 @@ static int fake_GetModuleFileNameW(HMODULE module, WCHAR *path, WORD nSize) { return MultiByteToWideChar(CP_UTF8, 0, (const char *)Tcl_GetHashValue(entry), -1 , path, nSize); } -static int fake_GetModuleFileNameA(HMODULE module, LPSTR lpFilename, WORD nSize) { +static DWORD fake_GetModuleFileNameA(HMODULE module, LPSTR lpFilename, WORD nSize) { Tcl_MutexLock(&dllDirectoryNameMutex); Tcl_HashEntry *entry = Tcl_FindHashEntry(&vfsPathTable, module); Tcl_MutexUnlock(&dllDirectoryNameMutex); @@ -421,7 +421,8 @@ static int fake_GetModuleFileNameA(HMODULE module, LPSTR lpFilename, WORD nSize) return GetModuleFileNameA(module, lpFilename, nSize); } strncpy(lpFilename, (const char *)Tcl_GetHashValue(entry), nSize); - return strlen(lpFilename); + lpFilename[nSize] = 0; + return (DWORD)strlen(lpFilename); } MODULE_SCOPE void * From b7abac85a2ebc1bc699d858074c796320e7a580f Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Mon, 2 Dec 2024 14:10:06 +0000 Subject: [PATCH 29/33] Add memorymodule constraint. Make win/dltest build work with minizip --- tests/memorymodule.test | 5 +++-- win/dltest/Makefile.in | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/memorymodule.test b/tests/memorymodule.test index 5f2195e4342..405817bec1d 100644 --- a/tests/memorymodule.test +++ b/tests/memorymodule.test @@ -4,6 +4,7 @@ if {"::tcltest" ni [namespace children]} { } testConstraint thread [expr {0 == [catch {package require Thread 2.7-}]}] +testConstraint memorymodule [expr {0 != [tcl::build-info memorymodule]}] testConstraint memorymoduletest [expr {0 == [catch {zipfs mount dltest/memorymoduletest.zip [file join [zipfs root] memorymoduletest]}]}] lappend auto_path [file join [zipfs root] memorymoduletest] @@ -13,12 +14,12 @@ test memorymodule-1.0 {info loaded} memorymoduletest { info loaded {} Memorymoduletest } [file join [zipfs root] memorymoduletest/tcl9memorymoduletest.dll] -test memorymodule-1.1 {GetModuleFileNameA (WIP)} memorymoduletest { +test memorymodule-1.1 {GetModuleFileNameA (WIP)} {memorymoduletest memorymodule} { package require memorymoduletest GetModuleFileNameA } //zipfs:/memorymoduletest/tcl9memorymoduletest.dll -test memorymodule-1.2 {GetModuleFileNameW (WIP)} memorymoduletest { +test memorymodule-1.2 {GetModuleFileNameW (WIP)} {memorymoduletest memorymodule} { package require memorymoduletest GetModuleFileNameW } //zipfs:/memorymoduletest/tcl9memorymoduletest.dll diff --git a/win/dltest/Makefile.in b/win/dltest/Makefile.in index 75a223c8e3a..c8cd25992fb 100644 --- a/win/dltest/Makefile.in +++ b/win/dltest/Makefile.in @@ -28,6 +28,9 @@ CC_SWITCHES = $(CFLAGS) -I${SRC_DIR}/../../generic \ all: memorymoduletest.zip memorymoduletest.zip: tcl9memorymoduletest${SHLIB_SUFFIX} + @if test -f ../minizip.exe; then \ + cp ../minizip.exe . ; \ + fi (zip=`(realpath '${NATIVE_ZIP}' || readlink -m '${NATIVE_ZIP}') 2>/dev/null || \ (echo '${NATIVE_ZIP}' | sed "s?^\./?$$(pwd)/?")`; \ $$zip memorymoduletest.zip pkgIndex.tcl *.dll >/dev/null && \ From 337b5b7a5bce491365c8d195cb4cb4c0f16682c9 Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Mon, 2 Dec 2024 15:40:55 +0000 Subject: [PATCH 30/33] Bring back Makefile's how they were --- win/Makefile.in | 9 ++------- win/makefile.vc | 11 +---------- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/win/Makefile.in b/win/Makefile.in index a913bad378b..e6245cecfff 100644 --- a/win/Makefile.in +++ b/win/Makefile.in @@ -547,15 +547,10 @@ ${TCL_ZIP_FILE}: ${ZIP_INSTALL_OBJS} ${DDE_DLL_FILE} ${REG_DLL_FILE} @mkdir -p ${TCL_VFS_PATH} @echo "creating ${TCL_VFS_PATH} (prepare compression)" @( \ - $(COPY) -a $(TOP_DIR)/library/* ${TCL_VFS_PATH}; \ - $(COPY) -a ${TCL_VFS_PATH}/manifest.txt ${TCL_VFS_PATH}/pkgIndex.tcl; \ - if test "${ZIPFS_BUILD}" != "2" ; then \ - $(COPY) ${DDE_DLL_FILE} ${TCL_VFS_PATH}/dde; \ - $(COPY) ${REG_DLL_FILE} ${TCL_VFS_PATH}/registry; \ - else \ + $(COPY) -a $(TOP_DIR)/library/* ${TCL_VFS_PATH}; \ + $(COPY) -a ${TCL_VFS_PATH}/manifest.txt ${TCL_VFS_PATH}/pkgIndex.tcl; \ rm -rf ${TCL_VFS_PATH}/dde; \ rm -rf ${TCL_VFS_PATH}/registry; \ - fi \ ) (zip=`(realpath '${NATIVE_ZIP}' || readlink -m '${NATIVE_ZIP}') 2>/dev/null || \ (echo '${NATIVE_ZIP}' | sed "s?^\./?$$(pwd)/?")`; \ diff --git a/win/makefile.vc b/win/makefile.vc index f19ba1adf98..6546d51b4bc 100644 --- a/win/makefile.vc +++ b/win/makefile.vc @@ -52,7 +52,7 @@ # 64-bit compiler, if your SDK has it. # # Basic macros and options usable on the commandline (see rules.vc for more info): -# OPTS=nomsvcrt,noembed,nothreads,pdbs,profile,static,symbols,thrdalloc,unchecked,none +# OPTS=memorymodule,nomsvcrt,noembed,nothreads,pdbs,profile,static,symbols,thrdalloc,unchecked,none # Sets special options for the core. The default is for none. # Any combination of the above may be used (comma separated). # 'none' will over-ride everything to nothing. @@ -674,16 +674,11 @@ $(TCLSCRIPTZIP): $(TCLLIB) $(TCLSH) dlls @$(MKDIR) "$(LIBTCLVFS)" @$(CPYDIR) $(LIBDIR) "$(LIBTCLVFS)\tcl_library" @move /y "$(LIBTCLVFS)\tcl_library\manifest.txt" "$(LIBTCLVFS)\tcl_library\pkgIndex.tcl" > NUL -!if $(STATIC_BUILD) # Remove the registry and dde directories as the DLLS are still external @del "$(LIBTCLVFS)\tcl_library\registry\pkgIndex.tcl" @rmdir "$(LIBTCLVFS)\tcl_library\registry" @del "$(LIBTCLVFS)\tcl_library\dde\pkgIndex.tcl" @rmdir "$(LIBTCLVFS)\tcl_library\dde" -!else - @$(COPY) $(TCLDDELIB) "$(LIBTCLVFS)\tcl_library\dde - @$(COPY) $(TCLREGLIB) "$(LIBTCLVFS)\tcl_library\registry -!endif @echo cd {$(OUT_DIR)} > "$(OUT_DIR)\zipper.tcl" @echo file delete -force {$(@F)} >> "$(OUT_DIR)\zipper.tcl" @echo zipfs mkzip {$(@F)} {$(LIBTCLVFSSUBDIR)} {$(LIBTCLVFSSUBDIR)} >> "$(OUT_DIR)\zipper.tcl" @@ -1146,17 +1141,13 @@ install-libraries: tclConfig tcl-nmake install-msgs install-tzdata "$(MODULE_INSTALL_DIR)\9.0\platform\shell-$(PKG_SHELL_VER).tm" !endif @echo Installing $(TCLDDELIBNAME) -!if !$(STATIC_BUILD) @$(CPY) "$(TCLDDELIB)" "$(LIB_INSTALL_DIR)\dde$(DDEDOTVERSION)\" @$(CPY) "$(ROOT)\library\dde\pkgIndex.tcl" \ "$(LIB_INSTALL_DIR)\dde$(DDEDOTVERSION)\" -!endif @echo Installing $(TCLREGLIBNAME) -!if !$(STATIC_BUILD) @$(CPY) "$(TCLREGLIB)" "$(LIB_INSTALL_DIR)\registry$(REGDOTVERSION)\" @$(CPY) "$(ROOT)\library\registry\pkgIndex.tcl" \ "$(LIB_INSTALL_DIR)\registry$(REGDOTVERSION)\" -!endif !if !$(TCL_EMBED_SCRIPTS) @echo Installing encodings @$(CPY) "$(ROOT)\library\encoding\*.enc" \ From 287d9f8cf3d58f22bbf09a2cf85114baad22992f Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Tue, 3 Dec 2024 11:15:13 +0000 Subject: [PATCH 31/33] This was only meant for testing --- generic/tclIOUtil.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/generic/tclIOUtil.c b/generic/tclIOUtil.c index 77bc1c0d699..279fa42ab79 100644 --- a/generic/tclIOUtil.c +++ b/generic/tclIOUtil.c @@ -3263,9 +3263,6 @@ Tcl_LoadFile( } mustCopyToTempAnyway: -#ifdef _WIN32 - Tcl_Panic("Copying to temp disabled"); // TODO!: Remove this when merging to trunk! -#endif #endif /* TCL_LOAD_FROM_MEMORY */ /* From 5c61c5bf12d9061acd1addbdcfb3f6586dbf1960 Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Tue, 3 Dec 2024 11:26:28 +0000 Subject: [PATCH 32/33] Bring back *.yml files from trunk, just with separate --enable-memorymodule build --- .github/workflows/onefiledist.yml | 78 ++++++++++++++++++++++++++++++- .github/workflows/win-build.yml | 24 +++++----- 2 files changed, 90 insertions(+), 12 deletions(-) diff --git a/.github/workflows/onefiledist.yml b/.github/workflows/onefiledist.yml index 521e4ed9fb2..4c8c6ac8938 100644 --- a/.github/workflows/onefiledist.yml +++ b/.github/workflows/onefiledist.yml @@ -49,6 +49,9 @@ jobs: with: name: Tclsh ${{ env.TCL_PATCHLEVEL }} Linux single-file build (snapshot) path: 1dist/*.tar + id: upload + outputs: + url: ${{ steps.upload.outputs.artifact-url }} macos: name: macOS runs-on: macos-13 @@ -116,6 +119,9 @@ jobs: with: name: Tclsh ${{ env.TCL_PATCHLEVEL }} macOS single-file build (snapshot) path: 1dist/*.dmg + id: upload + outputs: + url: ${{ steps.upload.outputs.artifact-url }} win: name: Windows runs-on: windows-2019 @@ -125,7 +131,7 @@ jobs: timeout-minutes: 10 env: CC: gcc - CFGOPT: --enable-memorymodule --disable-symbols --disable-shared + CFGOPT: --disable-symbols --disable-shared steps: - name: Install MSYS2 uses: msys2/setup-msys2@v2 @@ -165,3 +171,73 @@ jobs: with: name: Tclsh ${{ env.TCL_PATCHLEVEL }} Windows single-file build (snapshot) path: '1dist/*_snapshot.exe' + id: upload + outputs: + url: ${{ steps.upload.outputs.artifact-url }} + combine: + needs: + - linux + - macos + - win + name: Combine Artifacts (prototype) + runs-on: ubuntu-latest + defaults: + run: + shell: bash + timeout-minutes: 10 + env: + # See also + # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/store-information-in-variables + REMOTE_PATH: ${{ vars.PUBLISH_DROP_PATH }}/data-${{ github.sha }} + steps: + - name: Make directory + run: | + mkdir data + - name: Get Linux build + uses: actions/download-artifact@v4 + with: + path: data + # Can't download by artifact ID; stupid missing feature! + merge-multiple: true + - name: Check data downloaded + run: | + ls -AlR + working-directory: data + - name: Transfer built files + # https://github.com/marketplace/actions/rsync-deployments-action + uses: burnett01/rsync-deployments@7.0.1 + id: rsync + if: false # Disabled... for now + with: + # I don't know what the right switches are here, BTW + switches: -avzr + path: data/ + remote_path: ${{ env.REMOTE_PATH }} + remote_host: ${{ vars.PUBLISH_HOST }} + remote_user: ${{ vars.PUBLISH_USER }} + remote_key: ${{ secrets.DEPLOY_HOST_KEY }} + # MUST be a literal passwordless key + - name: Publish files + # https://github.com/marketplace/actions/ssh-remote-commands + uses: appleboy/ssh-action@v1.2.0 + id: ssh + if: steps.rsync.outcome == 'success' + with: + host: ${{ vars.PUBLISH_HOST }} + username: ${{ vars.PUBLISH_USER }} + key: ${{ secrets.DEPLOY_HOST_KEY }} + script: | + ${{ vars.PUBLISHER_SCRIPT }} ${{ env.REMOTE_PATH }} ${{ github.ref_type }} ${{ github.ref_name }} + - name: Report what would be done + if: steps.rsync.outcome == 'skipped' + env: + SWITCHES: -av + LOCAL_PATH: data/ + REMOTE_HOST: ${{ vars.PUBLISH_HOST }} + REMOTE_USER: ${{ vars.PUBLISH_USER }} + REMOTE_SCRIPT: | + ${{ vars.PUBLISHER_SCRIPT }} ${{ env.REMOTE_PATH }} ${{ github.ref_type }} ${{ github.ref_name }} + run: | + echo "would run: rsync $SWITCHES $LOCAL_PATH $REMOTE_USER@$REMOTE_HOST:$REMOTE_PATH" + echo "would run: ssh $REMOTE_USER@$REMOTE_HOST $REMOTE_SCRIPT" + # Consider https://github.com/marketplace/actions/slack-notify maybe? diff --git a/.github/workflows/win-build.yml b/.github/workflows/win-build.yml index 5e9475af95e..7bae2b09de8 100644 --- a/.github/workflows/win-build.yml +++ b/.github/workflows/win-build.yml @@ -21,12 +21,13 @@ jobs: strategy: matrix: config: + - "" + - "CHECKS=nodep" + - "OPTS=static" - "OPTS=memorymodule" - - "CHECKS=nodep OPTS=memorymodule" - - "OPTS=static,memorymodule" - - "OPTS=noembed,memorymodule" - - "OPTS=symbols,memorymodule" - - "OPTS=symbols,memorymodule STATS=compdbg,memdbg" + - "OPTS=noembed" + - "OPTS=symbols" + - "OPTS=symbols STATS=compdbg,memdbg" # Using powershell means we need to explicitly stop on failure steps: - name: Checkout @@ -65,13 +66,14 @@ jobs: strategy: matrix: config: + - "" + - "CFLAGS=-DTCL_NO_DEPRECATED=1" + - "--disable-shared" + - "--disable-zipfs" - "--enable-memorymodule" - - "--enable-memorymodule CFLAGS=-DTCL_NO_DEPRECATED=1" - - "--enable-memorymodule --disable-shared" - - "--enable-memorymodule --disable-zipfs" - - "--enable-memorymodule --enable-symbols" - - "--enable-memorymodule --enable-symbols=mem" - - "--enable-memorymodule --enable-symbols=all" + - "--enable-symbols" + - "--enable-symbols=mem" + - "--enable-symbols=all" # Using powershell means we need to explicitly stop on failure steps: - name: Install MSYS2 From 3c1a144a8335fe3f917ee8517294bd6be1af9541 Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Tue, 3 Dec 2024 11:31:35 +0000 Subject: [PATCH 33/33] Fix comment --- win/configure | 2 +- win/configure.ac | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/win/configure b/win/configure index 052b2ee69be..0d8579e1941 100755 --- a/win/configure +++ b/win/configure @@ -5267,7 +5267,7 @@ fi #-------------------------------------------------------------------- -# Zipfs support - Tip 430 +# MemoryModule support - Tip 708 #-------------------------------------------------------------------- # Check whether --enable-memorymodule was given. if test ${enable_memorymodule+y} diff --git a/win/configure.ac b/win/configure.ac index 35dcd8ef3ea..4f666f11804 100644 --- a/win/configure.ac +++ b/win/configure.ac @@ -210,7 +210,7 @@ AC_SUBST(INSTALL_MSGS) #-------------------------------------------------------------------- -# Zipfs support - Tip 430 +# MemoryModule support - Tip 708 #-------------------------------------------------------------------- AC_ARG_ENABLE(memorymodule, AS_HELP_STRING([--enable-memorymodule],