From f72ce398663eedd1e57ac3483781a5a5fc390752 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Thu, 14 Apr 2022 08:41:45 -0600 Subject: [PATCH] Update docs and usage of TEMPFILE/TempFileMunge tempfiles are now cleaned up via registering a cleanups with atexit(), instead of trying to squeeze removing the file into the command line. On Windows that caused the file to get deleted too early (did not work well with interactive mode), and on Linux it didn't remove the file at all. The Platform test expected to be able to read the tempfilename as the last argument of the "command", but this is no longer provided as the "rm filename" is no longer added, so now it has to chop off the prefix from the command-file argument to get the filename. Unrelatedly, two syntax warnings that turn up in the test output where some TeX syntax was listed in a docstring in a test are fixed by making that a raw string - got tired of seeing these. Fixes #4595 Signed-off-by: Mats Wichmann --- CHANGES.txt | 2 + RELEASE.txt | 2 + SCons/Platform/Platform.xml | 57 +++++++++++++++++++++----- SCons/Platform/PlatformTests.py | 35 +++++++++------- SCons/Platform/__init__.py | 44 ++++++++++---------- test/TEX/glossaries.py | 2 +- test/TEX/subdir_variantdir_include2.py | 2 +- 7 files changed, 96 insertions(+), 48 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 1ac704339e..2964a1cdc4 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -23,6 +23,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER the test fails. The rest is cleanup and type annotations. Be more careful that the returns from stderr() and stdout(), which *can* return None, are not used without checking. + - Temporary files created by TempFileMunge() are now cleaned up on + scons exit, instead of at the time they're used. Fixes #4595. RELEASE 4.8.1 - Tue, 03 Sep 2024 17:22:20 -0700 diff --git a/RELEASE.txt b/RELEASE.txt index 196103cfbc..fcc4f46a6f 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -36,6 +36,8 @@ FIXES if the variable is used on the command line with one of the enabling string as the value: the variable's default value is produced (previously it always produced True in this case). +- Temporary files created by TempFileMunge() are now cleaned up on + scons exit, instead of at the time they're used. Fixes #4595. IMPROVEMENTS ------------ diff --git a/SCons/Platform/Platform.xml b/SCons/Platform/Platform.xml index 0beafd1fdc..809cd9c431 100644 --- a/SCons/Platform/Platform.xml +++ b/SCons/Platform/Platform.xml @@ -298,16 +298,41 @@ The suffix used for shared object file names. -A callable object used to handle overly long command line strings, -since operations which call out to a shell will fail -if the line is longer than the shell can accept. -This tends to particularly impact linking. -The tempfile object stores the command line in a temporary -file in the appropriate format, and returns -an alternate command line so the invoked tool will make -use of the contents of the temporary file. -If you need to replace the default tempfile object, -the callable should take into account the settings of +Holds a callable object which will be invoked to transform long +command lines (string or list) into an alternate form. +Length limits on various operating systems +may cause long command lines to fail when calling out to +a shell to run the command. +Most often affects linking, when there are many object files and/or +libraries to be linked, but may also affect other +compilation steps which have many arguments. +&cv-TEMPFILE; is not called directly, +but rather is typically embedded in another +&consvar;, to be expanded when used. Example: + + + +env["TEMPFILE"] = TempFileMunge +env["LINKCOM"] = "${TEMPFILE('$LINK $TARGET $SOURCES', '$LINKCOMSTR')}" + + + +The SCons default value for &cv-TEMPFILE;, +TempFileMunge, +performs command substitution on the passed command line, +calculates whether modification is needed, +then puts all but the first word (assumed to be the command name) +of the resulting list into a temporary file +(sometimes called a response file or command file), +and returns a new command line consisting of the +the command name and an appropriately formatted reference +to the temporary file. + + + +A replacement for the default tempfile object would need +to do fundamentally the same thing, including taking into account +the values of &cv-link-MAXLINELENGTH;, &cv-link-TEMPFILEPREFIX;, &cv-link-TEMPFILESUFFIX;, @@ -315,6 +340,11 @@ the callable should take into account the settings of &cv-link-TEMPFILEDIR; and &cv-link-TEMPFILEARGESCFUNC;. +If a particular use case requires a different transformation +than the default, it is recommended to copy the mechanism and +define a new &consvar; and rewrite the relevant *COM +variable(s) to use it, to avoid possibly disrupting existing uses +of &cv-TEMPFILE;. @@ -324,6 +354,8 @@ and The prefix for the name of the temporary file used to store command lines exceeding &cv-link-MAXLINELENGTH;. +The prefix must include the compiler syntax to actually +include and process the file. The default prefix is '@', which works for the &MSVC; and GNU toolchains on Windows. Set this appropriately for other toolchains, @@ -338,7 +370,7 @@ or '-via' for ARM toolchain. The suffix for the name of the temporary file used to store command lines exceeding &cv-link-MAXLINELENGTH;. -The suffix should include the dot ('.') if one is wanted as +The suffix should include the dot ('.') if one is needed as it will not be added automatically. The default is .lnk. @@ -363,6 +395,9 @@ Note this value is used literally and not expanded by the subst logic. The directory to create the long-lines temporary file in. +If unset, some suitable default should be chosen. +The default tempfile object lets the &Python; +tempfile module choose. diff --git a/SCons/Platform/PlatformTests.py b/SCons/Platform/PlatformTests.py index c786aa7d8d..691aed2598 100644 --- a/SCons/Platform/PlatformTests.py +++ b/SCons/Platform/PlatformTests.py @@ -207,9 +207,7 @@ def test_MAXLINELENGTH(self) -> None: assert cmd != defined_cmd, cmd def test_TEMPFILEARGJOINBYTE(self) -> None: - """ - Test argument join byte TEMPFILEARGJOINBYTE - """ + """Test argument join byte TEMPFILEARGJOINBYTE.""" # Init class with cmd, such that the fully expanded # string reads "a test command line". @@ -232,19 +230,24 @@ def test_TEMPFILEARGJOINBYTE(self) -> None: SCons.Action.print_actions = 0 env['MAXLINELENGTH'] = len(expanded_cmd)-1 cmd = t(None, None, env, 0) - # print("CMD is:%s"%cmd) + # print(f"[CMD is: {cmd}]") - with open(cmd[-1],'rb') as f: + if cmd[-1].startswith('@'): + tempfile = cmd[-1][1:] + else: + tempfile = cmd[-1] + with open(tempfile, 'rb') as f: file_content = f.read() - # print("Content is:[%s]"%file_content) + # print(f"[Content of {tempfile} is:{file_content}]") # ...and restoring its setting. SCons.Action.print_actions = old_actions - assert file_content != env['TEMPFILEARGJOINBYTE'].join(['test','command','line']) + assert file_content != bytearray( + env['TEMPFILEARGJOINBYTE'].join(['test', 'command', 'line']), + encoding='utf-8', + ) def test_TEMPFILEARGESCFUNC(self) -> None: - """ - Test a custom TEMPFILEARGESCFUNC - """ + """Test a custom TEMPFILEARGESCFUNC.""" def _tempfile_arg_esc_func(arg): return str(arg).replace("line", "newarg") @@ -262,12 +265,16 @@ def _tempfile_arg_esc_func(arg): SCons.Action.print_actions = 0 env['TEMPFILEARGESCFUNC'] = _tempfile_arg_esc_func cmd = t(None, None, env, 0) - # print("CMD is: %s"%cmd) + # print(f"[CMD is: {cmd}]") - with open(cmd[-1], 'rb') as f: + if cmd[-1].startswith('@'): + tempfile = cmd[-1][1:] + else: + tempfile = cmd[-1] + with open(tempfile, 'rb') as f: file_content = f.read() - # print("Content is:[%s]"%file_content) - # # ...and restoring its setting. + # print(f"[Content of {tempfile} is:{file_content}]") + # ...and restoring its setting. SCons.Action.print_actions = old_actions assert b"newarg" in file_content diff --git a/SCons/Platform/__init__.py b/SCons/Platform/__init__.py index 77eea5c099..aec1a8989a 100644 --- a/SCons/Platform/__init__.py +++ b/SCons/Platform/__init__.py @@ -42,6 +42,7 @@ import SCons.compat +import atexit import importlib import os import sys @@ -150,7 +151,7 @@ class TempFileMunge: the length of command lines. Example:: env["TEMPFILE"] = TempFileMunge - env["LINKCOM"] = "${TEMPFILE('$LINK $TARGET $SOURCES','$LINKCOMSTR')}" + env["LINKCOM"] = "${TEMPFILE('$LINK $TARGET $SOURCES', '$LINKCOMSTR')}" By default, the name of the temporary file used begins with a prefix of '@'. This may be configured for other tool chains by @@ -258,31 +259,27 @@ def __call__(self, target, source, env, for_signature): fd, tmp = tempfile.mkstemp(suffix, dir=tempfile_dir, text=True) native_tmp = SCons.Util.get_native_path(tmp) + # arrange for cleanup on exit: + + def tmpfile_cleanup(file) -> None: + os.remove(file) + + atexit.register(tmpfile_cleanup, tmp) + if env.get('SHELL', None) == 'sh': # The sh shell will try to escape the backslashes in the # path, so unescape them. native_tmp = native_tmp.replace('\\', r'\\\\') - # In Cygwin, we want to use rm to delete the temporary - # file, because del does not exist in the sh shell. - rm = env.Detect('rm') or 'del' - else: - # Don't use 'rm' if the shell is not sh, because rm won't - # work with the Windows shells (cmd.exe or command.com) or - # Windows path names. - rm = 'del' if 'TEMPFILEPREFIX' in env: prefix = env.subst('$TEMPFILEPREFIX') else: - prefix = '@' + prefix = "@" tempfile_esc_func = env.get('TEMPFILEARGESCFUNC', SCons.Subst.quote_spaces) - args = [ - tempfile_esc_func(arg) - for arg in cmd[1:] - ] + args = [tempfile_esc_func(arg) for arg in cmd[1:]] join_char = env.get('TEMPFILEARGJOIN', ' ') - os.write(fd, bytearray(join_char.join(args) + "\n", 'utf-8')) + os.write(fd, bytearray(join_char.join(args) + "\n", encoding="utf-8")) os.close(fd) # XXX Using the SCons.Action.print_actions value directly @@ -301,15 +298,20 @@ def __call__(self, target, source, env, for_signature): # purity get in the way of just being helpful, so we'll # reach into SCons.Action directly. if SCons.Action.print_actions: - cmdstr = env.subst(self.cmdstr, SCons.Subst.SUBST_RAW, target, - source) if self.cmdstr is not None else '' + cmdstr = ( + env.subst(self.cmdstr, SCons.Subst.SUBST_RAW, target, source) + if self.cmdstr is not None + else '' + ) # Print our message only if XXXCOMSTR returns an empty string - if len(cmdstr) == 0 : - cmdstr = ("Using tempfile "+native_tmp+" for command line:\n"+ - str(cmd[0]) + " " + " ".join(args)) + if not cmdstr: + cmdstr = ( + f"Using tempfile {native_tmp} for command line:\n" + f'{cmd[0]} {" ".join(args)}' + ) self._print_cmd_str(target, source, env, cmdstr) - cmdlist = [cmd[0], prefix + native_tmp + '\n' + rm, native_tmp] + cmdlist = [cmd[0], prefix + native_tmp] # Store the temporary file command list into the target Node.attributes # to avoid creating two temporary files one for print and one for execute. diff --git a/test/TEX/glossaries.py b/test/TEX/glossaries.py index 069fcc3769..d418ee266e 100644 --- a/test/TEX/glossaries.py +++ b/test/TEX/glossaries.py @@ -23,7 +23,7 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -""" +r""" Validate that use of \glossaries in TeX source files causes SCons to be aware of the necessary created glossary files. diff --git a/test/TEX/subdir_variantdir_include2.py b/test/TEX/subdir_variantdir_include2.py index 953a2293e1..2b615f194d 100644 --- a/test/TEX/subdir_variantdir_include2.py +++ b/test/TEX/subdir_variantdir_include2.py @@ -23,7 +23,7 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -""" +r""" Verify that we execute TeX in a subdirectory (if that's where the document resides) by checking that all the auxiliary files get created there and not in the top-level directory. Test this when variantDir is used