Long path and Unicode support on Windows 10.0.1607/10.0.1903+ (#316)

Long path support is still not universal, even after an app has explicitly opted-in via manifest it also requires registry modification or group policies, see https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation?tabs=registry#enable-long-paths-in-windows-10-version-1607-and-later though Python and Git installers nudges to enable it by their installers for a while.

`-X utf8` forces Python to use UTF-8 when its outputs are piped (will be default in 3.15 https://peps.python.org/pep-0686/)

Also had to fix clang-linux and msvc toolsets manifest embedding issues.

.ci/azp-windows-test.yml

@@ -14,7 +14,7 @@ steps:
     $env:path += ';' + $env:CXX_PATH
     cd test
     echo "using" $env:TEST_TOOLSET ":" ":" $env:CXX ";" > ${env:HOME}/user-config.jam
-    py test_all.py $env:TEST_TOOLSET
+    py -X utf8 test_all.py $env:TEST_TOOLSET
     cd ..
   displayName: Test
 - powershell: |

.github/workflows/core_tests.yml

@@ -65,6 +65,7 @@ jobs:
 
       - name: Test
         working-directory: test
+        env: { PYTHONUTF8: 1 }
         run: ./test_all.py ${{matrix.toolset}}
 
       - name: 'No Warnings'

Jamroot.jam

@@ -150,6 +150,10 @@ exe b2
         <toolset>clang-win:<find-static-library>kernel32
         <toolset>clang-win:<find-static-library>advapi32
         <toolset>clang-win:<find-static-library>user32
+        <target-os>windows,<toolset>gcc:<source>src/engine/res.rc
+        <target-os>windows,<toolset>clang:<source>src/engine/res.rc
+        <toolset>msvc:<embed-manifest-file>src/engine/b2.exe.manifest
+        <toolset>clang-win:<embed-manifest-file>src/engine/b2.exe.manifest
     ;
 explicit b2 ;
 

appveyor.yml

@@ -32,6 +32,7 @@ environment:
     job_group: 'Test'
     TOOLSET: clang-win
     TEST_TOOLSET: clang-win
+    B2_DONT_EMBED_MANIFEST: true  # lld-link: error: unable to find mt.exe in PATH: no such file or directory
     APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
   - job_name: 'Cygwin 3.1.7 x64, Test'
     job_group: 'TestCygwin'
@@ -61,7 +62,7 @@ for:
           echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> TEST"
           echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
           cd test
-          py test_all.py %TEST_TOOLSET%
+          py -X utf8 test_all.py %TEST_TOOLSET%
           cd ..
       - cmd: |
           echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"

src/engine/b2.exe.manifest

@@ -0,0 +1,22 @@
+<assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0"
+          xmlns:asmv3="urn:schemas-microsoft-com:asm.v3">
+  <compatibility xmlns="urn:schemas-microsoft-com:compatibility.v1">
+    <application>
+      <!--Windows 7 -->
+      <supportedOS Id="{35138b9a-5d96-4fbd-8e2d-a2440225f93a}"/>
+      <!--Windows 8 -->
+      <supportedOS Id="{4a2f28e3-53b9-4441-ba9c-d69d4a4a6e38}"/>
+      <!--Windows 8.1 -->
+      <supportedOS Id="{1f676c76-80e1-4239-95bb-83d0f6d0da78}"/>
+      <!-- Windows 10 and 11 -->
+      <supportedOS Id="{8e0f7a12-bfb3-4fe8-b9a5-48fd50a15a9a}"/>
+    </application>
+  </compatibility>
+  <asmv3:application>
+    <asmv3:windowsSettings xmlns:ws1="http://schemas.microsoft.com/SMI/2016/WindowsSettings"
+                           xmlns:ws2="http://schemas.microsoft.com/SMI/2019/WindowsSettings">
+      <ws1:longPathAware>true</ws1:longPathAware>
+      <ws2:activeCodePage>UTF-8</ws2:activeCodePage>
+    </asmv3:windowsSettings>
+  </asmv3:application>
+</assembly>

src/engine/build.sh

@@ -503,6 +503,10 @@ mod_version.cpp \
     if test_true ${B2_DEBUG_OPT} ; then B2_CXXFLAGS="${B2_CXXFLAGS_DEBUG}"
     else B2_CXXFLAGS="${B2_CXXFLAGS_RELEASE} -DNDEBUG"
     fi
+    if [ -z "$B2_DONT_EMBED_MANIFEST" ] && [ -x "$(command -v windres)" ] ; then
+        B2_CXXFLAGS="${B2_CXXFLAGS} -Wl,res.o"
+        ( B2_VERBOSE_OPT=${TRUE} echo_run windres --input res.rc --output res.o )
+    fi
     ( B2_VERBOSE_OPT=${TRUE} echo_run ${B2_CXX} ${B2_CXXFLAGS} ${B2_SOURCES} -o b2 )
 }
 

src/engine/config_toolset.bat

@@ -39,7 +39,7 @@ if not "_%B2_TOOLSET_ROOT%_" == "__" (
 set "B2_CXX="%CXX%" /nologo /MP /MT /TP /Feb2 /wd4996 /O2 /GL /EHsc"
 set "B2_CXX_LINK=/link kernel32.lib advapi32.lib user32.lib"
 set "_known_=1"
-goto :eof
+goto :Embed_Minafest_Via_Link
 
 :Config_VC11
 if not defined CXX ( set "CXX=cl" )
@@ -54,7 +54,7 @@ if NOT "_%B2_TOOLSET_ROOT%_" == "__" (
 set "B2_CXX="%CXX%" /nologo /MP /MT /TP /Feb2 /wd4996 /O2 /GL /EHsc"
 set "B2_CXX_LINK=/link kernel32.lib advapi32.lib user32.lib"
 set "_known_=1"
-goto :eof
+goto :Embed_Minafest_Via_Link
 
 :Config_VC12
 if not defined CXX ( set "CXX=cl" )
@@ -73,7 +73,7 @@ if NOT "_%B2_TOOLSET_ROOT%_" == "__" (
 set "B2_CXX="%CXX%" /nologo /MP /MT /TP /Feb2 /wd4996 /O2 /GL /EHsc"
 set "B2_CXX_LINK=/link kernel32.lib advapi32.lib user32.lib"
 set "_known_=1"
-goto :eof
+goto :Embed_Minafest_Via_Link
 
 :Config_VC14
 if not defined CXX ( set "CXX=cl" )
@@ -93,7 +93,7 @@ if NOT "_%B2_TOOLSET_ROOT%_" == "__" (
 set "B2_CXX="%CXX%" /nologo /MP /MT /TP /Feb2 /wd4996 /O2 /GL /EHsc"
 set "B2_CXX_LINK=/link kernel32.lib advapi32.lib user32.lib"
 set "_known_=1"
-goto :eof
+goto :Embed_Minafest_Via_Link
 
 :Config_VC141
 if not defined CXX ( set "CXX=cl" )
@@ -115,7 +115,7 @@ popd
 set "B2_CXX="%CXX%" /nologo /MP /MT /TP /Feb2 /wd4996 /O2 /GL /EHsc"
 set "B2_CXX_LINK=/link kernel32.lib advapi32.lib user32.lib"
 set "_known_=1"
-goto :eof
+goto :Embed_Minafest_Via_Link
 
 :Config_VC142
 if not defined CXX ( set "CXX=cl" )
@@ -137,7 +137,7 @@ popd
 set "B2_CXX="%CXX%" /nologo /MP /MT /TP /Feb2 /wd4996 /O2 /GL /EHsc"
 set "B2_CXX_LINK=/link kernel32.lib advapi32.lib user32.lib"
 set "_known_=1"
-goto :eof
+goto :Embed_Minafest_Via_Link
 
 :Config_VC143
 if not defined CXX ( set "CXX=cl" )
@@ -160,7 +160,7 @@ popd
 set "B2_CXX="%CXX%" /nologo -TP /wd4996 /wd4675 /EHs /GR /Zc:throwingNew /O2 /Ob2 /W3 /MD /Zc:forScope /Zc:wchar_t /Zc:inline /Gw /favor:blend /Feb2"
 set "B2_CXX_LINK=/link kernel32.lib advapi32.lib user32.lib"
 set "_known_=1"
-goto :eof
+goto :Embed_Minafest_Via_Link
 
 :Config_VCUNK
 if NOT "_%B2_TOOLSET%_" == "_vcunk_" goto Skip_VCUNK
@@ -182,7 +182,7 @@ popd
 set "B2_CXX="%CXX%" /nologo /MP /MT /TP /Feb2 /wd4996 /O2 /GL /EHsc"
 set "B2_CXX_LINK=/link kernel32.lib advapi32.lib user32.lib"
 set "_known_=1"
-goto :eof
+goto :Embed_Minafest_Via_Link
 
 :Config_BORLAND
 if not defined CXX ( set "CXX=bcc32c" )
@@ -209,13 +209,13 @@ goto :eof
 if not defined CXX ( set "CXX=g++" )
 set "B2_CXX="%CXX%" -x c++ -std=c++11 -s -O3 -o b2.exe -D_GNU_SOURCE"
 set "_known_=1"
-goto :eof
+goto :Embed_Minafest_Via_Windres
 
 :Config_CLANG
 if not defined CXX ( set "CXX=clang++" )
 set "B2_CXX="%CXX%" -x c++ -std=c++11 -s -O3 -o b2.exe"
 set "_known_=1"
-goto :eof
+goto :Embed_Minafest_Via_Windres
 
 :Config_CLANG_WIN
 if not defined CXX ( set "CXX=clang-cl" )
@@ -224,10 +224,10 @@ if "_%ProgramFiles(x86)%_" == "__" (
 ) else (
     set "PATH=%PATH%;%ProgramFiles%\LLVM\bin;%ProgramFiles(x86)%\LLVM\bin"
 )
-set "B2_CXX="%CXX%" /MT /TP /Feb2 /wd4996 /O2 /EHsc /Gw /Zc:inline"
+set "B2_CXX="%CXX%" /MT /TP /Feb2 /wd4996 /O2 /EHsc /Gw /Zc:inline -fuse-ld=lld"
 set "B2_CXX_LINK=/link kernel32.lib advapi32.lib user32.lib"
 set "_known_=1"
-goto :eof
+goto :Embed_Minafest_Via_Link
 
 :Config_GCC_NOCYGWIN
 if not defined CXX ( set "CXX=g++" )
@@ -249,4 +249,17 @@ if not "_%B2_TOOLSET_ROOT%_" == "__" (
 for /F "delims=" %%I in ("%CXX%") do set "PATH=%PATH%;%%~dpI"
 set "B2_CXX="%CXX%" -x c++ -std=c++11 -s -O3 -static -o b2.exe"
 set "_known_=1"
+goto :Embed_Minafest_Via_Windres
+
+:Embed_Minafest_Via_Link
+if not defined B2_DONT_EMBED_MANIFEST (
+    set "B2_CXX_LINK=%B2_CXX_LINK% /MANIFEST:EMBED /MANIFESTINPUT:b2.exe.manifest"
+)
+goto :eof
+
+:Embed_Minafest_Via_Windres
+if not defined B2_DONT_EMBED_MANIFEST (
+    where windres >NUL 2>NUL
+    if %ERRORLEVEL% NEQ 0 ( call; ) else ( set "B2_CXX=windres --input res.rc --output res.o && %B2_CXX% -Wl,res.o" )
+)
 goto :eof

src/engine/jam.cpp

@@ -152,6 +152,11 @@
 # include <signal.h>
 #endif
 
+#ifdef WIN32
+# define WIN32_LEAN_AND_MEAN
+# include <windows.h>
+#endif
+
 struct globs globs =
 {
     0,          /* noexec */
@@ -614,6 +619,44 @@ int guarded_main( int argc, char * * argv )
     return status ? EXITBAD : EXITOK;
 }
 
+#ifdef WIN32
+namespace {
+
+struct SetConsoleCodepage
+{
+    SetConsoleCodepage()
+    {
+        // Check whether UTF-8 is actually the default encoding for this process
+        if (GetACP() != CP_UTF8)
+            return;
+
+        orig_console_cp = GetConsoleCP();
+        if (orig_console_cp != 0 && orig_console_cp != CP_UTF8)
+            SetConsoleCP(CP_UTF8);
+        orig_console_output_cp = GetConsoleOutputCP();
+        if (orig_console_output_cp != 0 && orig_console_output_cp != CP_UTF8)
+            SetConsoleOutputCP(CP_UTF8);
+    }
+
+    ~SetConsoleCodepage()
+    {
+        // Restore original console codepage
+        if (orig_console_cp != 0 && orig_console_cp != CP_UTF8)
+            SetConsoleCP(orig_console_cp);
+        if (orig_console_output_cp != 0 && orig_console_output_cp != CP_UTF8)
+            SetConsoleOutputCP(orig_console_output_cp);
+    }
+
+private:
+    UINT orig_console_cp = 0;
+    UINT orig_console_output_cp = 0;
+};
+
+static const SetConsoleCodepage g_console_codepage_setter{};
+
+}
+#endif
+
 int main( int argc, char * * argv )
 {
     BJAM_MEM_INIT();

src/engine/res.rc

@@ -0,0 +1 @@
+1 24 b2.exe.manifest

src/tools/clang-linux.jam

@@ -21,6 +21,7 @@ import type ;
 import numbers ;
 import os ;
 import property ;
+import rc ;
 import set ;
 
 feature.extend-subfeature toolset clang : platform : linux ;
@@ -86,6 +87,23 @@ rule init ( version ? :  command * : options * ) {
     archiver = $(root)/bin/ar ;
     }
   toolset.flags clang-linux.archive .AR $(condition) : $(archiver[1]) ;
+
+    # - Resource compiler.
+    local rc = [ common.get-invocation-command-nodefault clang-linux : windres :
+        [ feature.get-values <rc> : $(options) ] : $(bin) : search-path ] ;
+    local rc-type = [ feature.get-values <rc-type> : $(options) ] ;
+    rc-type ?= windres ;
+    if ! $(rc)
+    {
+        # If we can not find an RC compiler we fallback to a null one that
+        # creates empty object files. This allows the same Jamfiles to work
+        # across the board. The null RC uses assembler to create the empty
+        # objects, so configure that.
+        rc = [ common.get-invocation-command clang-linux : as : : $(bin) : search-path ]
+            ;
+        rc-type = null ;
+    }
+    rc.configure $(rc) : $(condition) : <rc-type>$(rc-type) ;
 }
 
 rule get-full-version ( command-string )

src/tools/msvc.jam

@@ -2000,6 +2000,7 @@ local rule register-toolset-really ( )
         toolset.flags msvc.link LINKFLAGS <debug-symbols>on : /DEBUG ;
         toolset.flags msvc.link DEF_FILE <def-file> ;
         toolset.flags msvc.link MANIFEST_FILE <embed-manifest-via>linker : <embed-manifest-file> ;
+        toolset.flags msvc.link EMBED_MANIFEST_FILE <embed-manifest-via>mt : <embed-manifest-file> ;
 
         # The linker disables the default optimizations when using /DEBUG so we
         # have to enable them manually for release builds with debug symbols.

test/BoostBuild.py

@@ -332,6 +332,12 @@ def __init__(self, arguments=None, executable=None,
 
         os.chdir(self.workdir)
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type, value, traceback):
+        self.cleanup()
+
     def cleanup(self):
         try:
             TestCmd.TestCmd.cleanup(self)
@@ -383,11 +389,13 @@ def write(self, file, content, wait=True):
         self.__makedirs(os.path.dirname(nfile), wait)
         if not type(content) == bytes:
             content = content.encode()
-        f = open(nfile, "wb")
         try:
-            f.write(content)
-        finally:
-            f.close()
+            with open(nfile, "wb") as f:
+                f.write(content)
+        except Exception as e:
+            annotation("failure", f"Could not create file '{nfile}': {e}")
+            annotate_stack_trace(level=3)
+            self.fail_test(1)
         self.__ensure_newer_than_last_build(nfile)
 
     def rename(self, src, dst):
@@ -1049,9 +1057,11 @@ def __makedirs(self, path, wait):
                     os.mkdir(path)
                     self.__ensure_newer_than_last_build(path)
             else:
-                os.makedirs(path)
-        except Exception:
-            pass
+                os.makedirs(path, exist_ok=True)
+        except Exception as e:
+            annotation("failure", f"Could not create path '{path}': {e}")
+            annotate_stack_trace(level=3)
+            self.fail_test(1)
 
     def __python_timestamp_resolution(self, path, minimum_resolution):
         """