Merge pull request #4597 from mwichmann/issue/TEMPFILE-fiddle

Update docs and usage of TEMPFILE/TempFileMunge

CHANGES.txt

@@ -23,6 +23,8 @@ RELEASE  VERSION/DATE TO BE FILLED IN LATER
       the test fails. The rest is cleanup and type annotations. Be more
       careful that the returns from stderr() and stdout(), which *can*
       return None, are not used without checking.
+    - Temporary files created by TempFileMunge() are now cleaned up on
+      scons exit, instead of at the time they're used.  Fixes #4595.
     - Override envirionments, created when giving construction environment
       keyword arguments to Builder calls (or manually, through the undocumented
       Override method), were modified not to "leak" on item deletion.  The item

RELEASE.txt

@@ -41,6 +41,8 @@ FIXES
   if the variable is used on the command line with one of the enabling
   string as the value: the variable's default value is produced (previously
   it always produced True in this case).
+- Temporary files created by TempFileMunge() are now cleaned up on
+  scons exit, instead of at the time they're used.  Fixes #4595.
 
 IMPROVEMENTS
 ------------

SCons/Platform/Platform.xml

@@ -298,23 +298,53 @@ The suffix used for shared object file names.
 <cvar name="TEMPFILE">
 <summary>
 <para>
-A callable object used to handle overly long command line strings,
-since operations which call out to a shell will fail
-if the line is longer than the shell can accept.
-This tends to particularly impact linking.
-The tempfile object stores the command line in a temporary
-file in the appropriate format, and returns
-an alternate command line so the invoked tool will make
-use of the contents of the temporary file.
-If you need to replace the default tempfile object,
-the callable should take into account the settings of
+Holds a callable object which will be invoked to transform long
+command lines (string or list) into an alternate form.
+Length limits on various operating systems
+may cause long command lines to fail when calling out to
+a shell to run the command.
+Most often affects linking, when there are many object files and/or
+libraries to be linked, but may also affect other
+compilation steps which have many arguments.
+&cv-TEMPFILE; is not called directly,
+but rather is typically embedded in another
+&consvar;, to be expanded when used. Example:
+</para>
+
+<example_commands>
+env["TEMPFILE"] = TempFileMunge
+env["LINKCOM"] = "${TEMPFILE('$LINK $TARGET $SOURCES', '$LINKCOMSTR')}"
+</example_commands>
+
+<para>
+The SCons default value for &cv-TEMPFILE;,
+<classname>TempFileMunge</classname>,
+performs command substitution on the passed command line,
+calculates whether modification is needed,
+then puts all but the first word (assumed to be the command name)
+of the resulting list into a temporary file
+(sometimes called a response file or command file),
+and returns a new command line consisting of the
+the command name and an appropriately formatted reference
+to the temporary file.
+</para>
+
+<para>
+A replacement for the default tempfile object would need
+to do fundamentally the same thing, including taking into account
+the values of
 &cv-link-MAXLINELENGTH;,
 &cv-link-TEMPFILEPREFIX;,
 &cv-link-TEMPFILESUFFIX;,
 &cv-link-TEMPFILEARGJOIN;,
 &cv-link-TEMPFILEDIR;
 and
 &cv-link-TEMPFILEARGESCFUNC;.
+If a particular use case requires a different transformation
+than the default, it is recommended to copy the mechanism and
+define a new &consvar; and rewrite the relevant <literal>*COM</literal>
+variable(s) to use it, to avoid possibly disrupting existing uses
+of &cv-TEMPFILE;.
 </para>
 </summary>
 </cvar>
@@ -324,6 +354,8 @@ and
 <para>
 The prefix for the name of the temporary file used
 to store command lines exceeding &cv-link-MAXLINELENGTH;.
+The prefix must include the compiler syntax to actually
+include and process the file.
 The default prefix is <literal>'@'</literal>, which works for the &MSVC;
 and GNU toolchains on Windows.
 Set this appropriately for other toolchains,
@@ -338,7 +370,7 @@ or <literal>'-via'</literal> for ARM toolchain.
 <para>
 The suffix for the name of the temporary file used
 to store command lines exceeding &cv-link-MAXLINELENGTH;.
-The suffix should include the dot ('.') if one is wanted as
+The suffix should include the dot ('.') if one is needed as
 it will not be added automatically.
 The default is <filename>.lnk</filename>.
 </para>
@@ -363,6 +395,9 @@ Note this value is used literally and not expanded by the subst logic.
 <summary>
 <para>
 The directory to create the long-lines temporary file in.
+If unset, some suitable default should be chosen.
+The default tempfile object lets the &Python;
+<systemitem>tempfile</systemitem> module choose.
 </para>
 </summary>
 </cvar>

SCons/Platform/PlatformTests.py

@@ -207,9 +207,7 @@ def test_MAXLINELENGTH(self) -> None:
         assert cmd != defined_cmd, cmd
 
     def test_TEMPFILEARGJOINBYTE(self) -> None:
-        """
-        Test argument join byte TEMPFILEARGJOINBYTE
-        """
+        """Test argument join byte TEMPFILEARGJOINBYTE."""
 
         # Init class with cmd, such that the fully expanded
         # string reads "a test command line".
@@ -232,19 +230,24 @@ def test_TEMPFILEARGJOINBYTE(self) -> None:
         SCons.Action.print_actions = 0
         env['MAXLINELENGTH'] = len(expanded_cmd)-1
         cmd = t(None, None, env, 0)
-        # print("CMD is:%s"%cmd)
+        # print(f"[CMD is: {cmd}]")
 
-        with open(cmd[-1],'rb') as f:
+        if cmd[-1].startswith('@'):
+            tempfile = cmd[-1][1:]
+        else:
+            tempfile = cmd[-1]
+        with open(tempfile, 'rb') as f:
             file_content = f.read()
-        # print("Content is:[%s]"%file_content)
+        # print(f"[Content of {tempfile} is:{file_content}]")
         # ...and restoring its setting.
         SCons.Action.print_actions = old_actions
-        assert file_content != env['TEMPFILEARGJOINBYTE'].join(['test','command','line'])
+        assert file_content != bytearray(
+            env['TEMPFILEARGJOINBYTE'].join(['test', 'command', 'line']),
+            encoding='utf-8',
+        )
 
     def test_TEMPFILEARGESCFUNC(self) -> None:
-        """
-        Test a custom TEMPFILEARGESCFUNC
-        """
+        """Test a custom TEMPFILEARGESCFUNC."""
 
         def _tempfile_arg_esc_func(arg):
             return str(arg).replace("line", "newarg")
@@ -262,12 +265,16 @@ def _tempfile_arg_esc_func(arg):
         SCons.Action.print_actions = 0
         env['TEMPFILEARGESCFUNC'] = _tempfile_arg_esc_func
         cmd = t(None, None, env, 0)
-        # print("CMD is: %s"%cmd)
+        # print(f"[CMD is: {cmd}]")
 
-        with open(cmd[-1], 'rb') as f:
+        if cmd[-1].startswith('@'):
+            tempfile = cmd[-1][1:]
+        else:
+            tempfile = cmd[-1]
+        with open(tempfile, 'rb') as f:
             file_content = f.read()
-        # print("Content is:[%s]"%file_content)
-        # # ...and restoring its setting.
+        # print(f"[Content of {tempfile} is:{file_content}]")
+        # ...and restoring its setting.
         SCons.Action.print_actions = old_actions
         assert b"newarg" in file_content
 

SCons/Platform/__init__.py

@@ -42,6 +42,7 @@
 
 import SCons.compat
 
+import atexit
 import importlib
 import os
 import sys
@@ -150,7 +151,7 @@ class TempFileMunge:
     the length of command lines. Example::
 
         env["TEMPFILE"] = TempFileMunge
-        env["LINKCOM"] = "${TEMPFILE('$LINK $TARGET $SOURCES','$LINKCOMSTR')}"
+        env["LINKCOM"] = "${TEMPFILE('$LINK $TARGET $SOURCES', '$LINKCOMSTR')}"
 
     By default, the name of the temporary file used begins with a
     prefix of '@'.  This may be configured for other tool chains by
@@ -258,31 +259,27 @@ def __call__(self, target, source, env, for_signature):
         fd, tmp = tempfile.mkstemp(suffix, dir=tempfile_dir, text=True)
         native_tmp = SCons.Util.get_native_path(tmp)
 
+        # arrange for cleanup on exit:
+
+        def tmpfile_cleanup(file) -> None:
+            os.remove(file)
+
+        atexit.register(tmpfile_cleanup, tmp)
+
         if env.get('SHELL', None) == 'sh':
             # The sh shell will try to escape the backslashes in the
             # path, so unescape them.
             native_tmp = native_tmp.replace('\\', r'\\\\')
-            # In Cygwin, we want to use rm to delete the temporary
-            # file, because del does not exist in the sh shell.
-            rm = env.Detect('rm') or 'del'
-        else:
-            # Don't use 'rm' if the shell is not sh, because rm won't
-            # work with the Windows shells (cmd.exe or command.com) or
-            # Windows path names.
-            rm = 'del'
 
         if 'TEMPFILEPREFIX' in env:
             prefix = env.subst('$TEMPFILEPREFIX')
         else:
-            prefix = '@'
+            prefix = "@"
 
         tempfile_esc_func = env.get('TEMPFILEARGESCFUNC', SCons.Subst.quote_spaces)
-        args = [
-            tempfile_esc_func(arg)
-            for arg in cmd[1:]
-        ]
+        args = [tempfile_esc_func(arg) for arg in cmd[1:]]
         join_char = env.get('TEMPFILEARGJOIN', ' ')
-        os.write(fd, bytearray(join_char.join(args) + "\n", 'utf-8'))
+        os.write(fd, bytearray(join_char.join(args) + "\n", encoding="utf-8"))
         os.close(fd)
 
         # XXX Using the SCons.Action.print_actions value directly
@@ -301,15 +298,20 @@ def __call__(self, target, source, env, for_signature):
         # purity get in the way of just being helpful, so we'll
         # reach into SCons.Action directly.
         if SCons.Action.print_actions:
-            cmdstr = env.subst(self.cmdstr, SCons.Subst.SUBST_RAW, target,
-                               source) if self.cmdstr is not None else ''
+            cmdstr = (
+                env.subst(self.cmdstr, SCons.Subst.SUBST_RAW, target, source)
+                if self.cmdstr is not None
+                else ''
+            )
             # Print our message only if XXXCOMSTR returns an empty string
-            if len(cmdstr) == 0 :
-                cmdstr = ("Using tempfile "+native_tmp+" for command line:\n"+
-                    str(cmd[0]) + " " + " ".join(args))
+            if not cmdstr:
+                cmdstr = (
+                    f"Using tempfile {native_tmp} for command line:\n"
+                    f'{cmd[0]} {" ".join(args)}'
+                )
                 self._print_cmd_str(target, source, env, cmdstr)
 
-        cmdlist = [cmd[0], prefix + native_tmp + '\n' + rm, native_tmp]
+        cmdlist = [cmd[0], prefix + native_tmp]
 
         # Store the temporary file command list into the target Node.attributes
         # to avoid creating two temporary files one for print and one for execute.

test/TEX/glossaries.py

@@ -23,7 +23,7 @@
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-"""
+r"""
 Validate that use of \glossaries in TeX source files causes SCons to
 be aware of the necessary created glossary files.
 

test/TEX/subdir_variantdir_include2.py

@@ -23,7 +23,7 @@
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-"""
+r"""
 Verify that we execute TeX in a subdirectory (if that's where the document
 resides) by checking that all the auxiliary files get created there and
 not in the top-level directory. Test this when variantDir is used