From 642222ef43bdc5a04d1525f077029a432fa1c014 Mon Sep 17 00:00:00 2001 From: Nikita Kniazev Date: Sun, 7 May 2023 04:00:02 +0300 Subject: [PATCH] emscripten renovation * inherit from clang toolset * update for 'new' fastcomp backend * exceptions support * dynamic linking support * pthread support * run-tests launcher via nodejs --- src/tools/emscripten.jam | 190 +++++++++++++++++++----------- src/tools/features/os-feature.jam | 1 + src/tools/gcc.jam | 4 +- test/BoostBuild.py | 42 ++++--- test/conditionals.py | 6 +- test/unit_test.py | 2 +- 6 files changed, 151 insertions(+), 94 deletions(-) diff --git a/src/tools/emscripten.jam b/src/tools/emscripten.jam index d6594c5e43..9faf2f7981 100644 --- a/src/tools/emscripten.jam +++ b/src/tools/emscripten.jam @@ -1,3 +1,4 @@ +# Copyright Nikita Kniazev # Copyright Rene Rivera 2016 # Distributed under the Boost Software License, Version 1.0. # (See accompanying file LICENSE.txt @@ -7,100 +8,151 @@ import feature ; import os ; import toolset ; import common ; -import gcc ; import type ; +import version ; + +feature.extend toolset : emscripten ; feature.feature embind : off on : propagated ; feature.feature closure : off on full : propagated ; feature.feature link-optimization : off on full : propagated ; +feature.subfeature debug-symbols : source-map : on : optional propagated ; +feature.subfeature exception-handling : demangle-support : on : optional propagated ; +feature.subfeature exception-handling : method : js : optional propagated link-incompatible ; + +if [ MATCH (--debug-(emscripten-)?configuration) : [ modules.peek : ARGV ] ] +{ + local rule .debug-configuration ( messages * ) + { + ECHO "notice: [emscripten-cfg]" $(messages) ; + } +} +else +{ + local rule .debug-configuration ( messages * ) { } +} -rule init ( version ? : command * : options * ) +rule init ( version ? : command * : options * ) { + # On Windows emcc calls emcc.bat but B2 will trip in CreateProcess if we + # let it call emcc because it does not work like a shell. + # We could make it call emcc.bat, but then it will trip on other issue, + # see https://github.com/bfgroup/b2/issues/309 + if ! $(command) && [ os.name ] = NT { + command = [ common.get-absolute-tool-path emcc ] ; + if $(command) { + local python = [ os.environ EMSDK_PYTHON ] ; + python ?= py ; + command = $(python) $(command)\\emcc.py ; + } + } + command = [ common.get-invocation-command emscripten : emcc : $(command) ] ; # Determine the version - if $(command) - { - local command-string = \"$(command)\" ; - command-string = $(command-string:J=" ") ; - version ?= [ MATCH "([0-9.]+)" - : [ SHELL "$(command-string) --version" ] ] ; - } + local command-string = [ common.make-command-string $(command) ] ; + version = [ MATCH "([0-9.]+)" : [ SHELL "$(command-string) --version" ] ] ; local condition = [ common.check-init-parameters emscripten : version $(version) ] ; + local conditions = [ feature.split $(condition) ] ; common.handle-options emscripten : $(condition) : $(command) : $(options) ; -} + .debug-configuration $(condition) ":: emcc ::" $(command) ; -feature.extend toolset : emscripten ; + local ar = [ feature.get-values : $(options) ] ; + ar ?= $(command[1--2]) $(command[-1]:B=emar) ; + ar = [ common.get-invocation-command emscripten : emar : $(ar) ] ; + toolset.flags emscripten.archive .AR $(condition) : $(ar) ; + .debug-configuration $(condition) ":: archiver ::" $(ar) ; -toolset.inherit-generators emscripten emscripten - : gcc - : gcc.mingw.link gcc.mingw.link.dll gcc.compile.c.pch gcc.compile.c++.pch - ; -toolset.inherit-rules emscripten : gcc ; -toolset.inherit-flags emscripten : gcc - : - off speed space - minimal debug - off on - off on - off on - ; - -type.set-generated-target-suffix EXE : emscripten : "js" ; -type.set-generated-target-suffix OBJ : emscripten : "bc" ; -type.set-generated-target-suffix STATIC_LIB : emscripten : "bc" ; - -toolset.flags emscripten.compile OPTIONS ; -toolset.flags emscripten.compile OPTIONS ; -toolset.flags emscripten.compile.c++ OPTIONS ; - -toolset.flags emscripten.compile OPTIONS off : -O0 ; -toolset.flags emscripten.compile OPTIONS speed : -O3 ; -toolset.flags emscripten.compile OPTIONS space : -Oz ; -toolset.flags emscripten.link OPTIONS off : -O0 ; -toolset.flags emscripten.link OPTIONS speed : -O3 ; -toolset.flags emscripten.link OPTIONS space : -O3 ; + local ProgramFiles ; + if [ os.name ] = NT { + ProgramFiles = [ os.environ "ProgramFiles" ] [ os.environ "ProgramFiles(x86)" ] ; + } + local nodejs = [ feature.get-values : $(options) ] ; + nodejs = [ common.get-invocation-command-nodefault emscripten : node : $(nodejs) : "$(ProgramFiles)\\nodejs" ] ; + if $(nodejs) { + local command-string = [ common.make-command-string $(nodejs) ] ; + local node-version = [ MATCH "v([0-9]+)" : [ SHELL "$(command-string) --version" ] ] ; + .debug-configuration $(condition) ":: nodejs version is" $(node-version:J=.) ; + if [ version.version-less $(node-version:E=0) : 16 ] { + toolset.add-defaults $(conditions:J=,)\:js ; + } + local result = [ SHELL "$(command-string) --version --experimental-wasm-threads" : exit-status no-output ] ; + if $(result[2]) = 0 { + nodejs += --experimental-wasm-threads ; + } + } + nodejs ?= nodejs ; + import testing ; + toolset.flags testing LAUNCHER $(condition) : \"$(nodejs)\" : unchecked ; + .debug-configuration $(condition) ":: nodejs ::" $(nodejs) ; + + version = [ SPLIT_BY_CHARACTERS $(version:E=0) : . ] ; + # The version number is my guess, I could not find when -pthread or -fwasm-exceptions were added + if [ version.version-less $(version[1]) : 2 ] { + toolset.add-requirements $(conditions:J=,)\:js ; + toolset.flags emscripten.compile OPTIONS $(condition)/multi : -sUSE_PTHREADS ; + } + if ! [ version.version-less $(version[1]) : 3 ] { + # bring back unsupported in v2 flags + toolset.flags emscripten.link FINDLIBS-ST-PFX $(condition)/shared : -Wl,-Bstatic ; + toolset.flags emscripten.link FINDLIBS-SA-PFX $(condition)/shared : -Wl,-Bdynamic ; + } +} -toolset.flags emscripten.compile OPTIONS on : --profiling-funcs ; +import clang-linux ; +toolset.inherit-generators emscripten : clang-linux ; +toolset.inherit-rules emscripten : clang-linux ; +toolset.inherit-flags emscripten : clang-linux : : + # emscripten barks on them being unsupported + RPATH_LINK + RPATH_OPTION + SONAME_OPTION + # supported only in v3, we reenable them conditionally in init + FINDLIBS-ST-PFX + FINDLIBS-SA-PFX + ; + +toolset.add-defaults emscripten:unknown ; +# dynamic linking is experemental and buggy +toolset.add-defaults emscripten:static ; + +# Emscripten can produce different kinds of .js and .wasm outputs: +# -o .wasm produces a standalone .wasm executable. +# -o .js produces a .wasm executable that uses emscripten runtime and a .js launcher. +# -o .js -sSTANDALONE_WASM produces a standalone .wasm executable and a .js launcher. +type.set-generated-target-suffix EXE : emscripten : js ; + +toolset.flags emscripten.compile.c++ OPTIONS on/js : -fexceptions ; +toolset.flags emscripten.link OPTIONS on/js : -fexceptions ; +toolset.flags emscripten.compile.c++ OPTIONS on/ : -fwasm-exceptions ; +toolset.flags emscripten.link OPTIONS on/ : -fwasm-exceptions ; + +# Emscripten support for shared libraries is incomplete, linker embeds direct +# dependencies into executable itself but do not embed transitive dependencies. +# We probably could workaround that in a custom linking generator which will add +# transitive dependencies via --preload-file/--embed-file, but it is a lot of work +# for a niche feature which needs to be fixed in the linker. +# There is -sNODERAWFS but LD_LIBRARY_PATH seems to only work for explicit dlopen. +import testing ; +# TODO: This is brittle and ugly, but currently there is no other way to specify flags for produced types by linker +local EXE_PRODUCING_TARGET_TYPES = EXE RUN_OUTPUT RUN RUN_FAIL UNIT_TEST ; +toolset.flags emscripten.link OPTIONS shared/$(EXE_PRODUCING_TARGET_TYPES) : -sMAIN_MODULE ; +toolset.flags emscripten.link OPTIONS shared/LIB : -sSIDE_MODULE ; +toolset.flags emscripten.link OPTIONS SHARED_LIB : -sSIDE_MODULE ; -toolset.flags emscripten.compile OPTIONS off : -fno-inline ; -toolset.flags emscripten.compile OPTIONS on : -Wno-inline ; -toolset.flags emscripten.compile OPTIONS full : -Wno-inline ; +toolset.flags emscripten.compile OPTIONS on : --profiling-funcs ; -toolset.flags emscripten OPTIONS off : -g0 ; -toolset.flags emscripten OPTIONS on : -g4 -s DEMANGLE_SUPPORT=1 ; -toolset.flags emscripten OPTIONS off : -fno-rtti ; +toolset.flags emscripten.link OPTIONS on/on : -sDEMANGLE_SUPPORT ; +toolset.flags emscripten.link OPTIONS on/on : -gsource-map ; toolset.flags emscripten.link OPTIONS on : --bind ; toolset.flags emscripten.link OPTIONS on : --closure 1 ; toolset.flags emscripten.link OPTIONS full : --closure 2 ; -toolset.flags emscripten.link OPTIONS off : --llvm-lto 0 ; +# things for old fastcomp backend which was removed in 2.0.0 (08/10/2020) toolset.flags emscripten.link OPTIONS on : --llvm-lto 1 ; toolset.flags emscripten.link OPTIONS full : --llvm-lto 3 ; - -actions compile.c -{ - "$(CONFIG_COMMAND)" -x c $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)" -} - -actions compile.c++ -{ - "$(CONFIG_COMMAND)" -x c++ $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)" -} - -actions archive -{ - "$(CONFIG_COMMAND)" $(AROPTIONS) -r -o "$(<)" "$(>)" -} - -toolset.flags emscripten.link USER_OPTIONS ; - -actions link bind LIBRARIES -{ - "$(CONFIG_COMMAND)" $(USER_OPTIONS) -L"$(LINKPATH)" -o "$(<)" "$(>)" "$(LIBRARIES)" $(START-GROUP) $(FINDLIBS-ST-PFX) -l$(FINDLIBS-ST) $(FINDLIBS-SA-PFX) -l$(FINDLIBS-SA) $(END-GROUP) $(OPTIONS) -} diff --git a/src/tools/features/os-feature.jam b/src/tools/features/os-feature.jam index ced1fc1ff8..e372380148 100644 --- a/src/tools/features/os-feature.jam +++ b/src/tools/features/os-feature.jam @@ -8,6 +8,7 @@ import modules ; import os ; .os-names = + unknown aix android appletv bsd cygwin darwin freebsd haiku hpux iphone linux netbsd openbsd osf qnx qnxnto sgi solaris unix unixware windows vms vxworks freertos diff --git a/src/tools/gcc.jam b/src/tools/gcc.jam index 834f5e1bf6..90948f5622 100644 --- a/src/tools/gcc.jam +++ b/src/tools/gcc.jam @@ -1035,12 +1035,12 @@ rule link.dll ( targets * : sources * : properties * ) actions link bind LIBRARIES { - "$(CONFIG_COMMAND)" @($(<[1]:T).rsp:O=FC:<=@":>=":E=-L"$(LINKPATH)" -Wl,$(RPATH_OPTION:E=-R)$(SPACE)-Wl,$(RPATH) -Wl,-rpath-link$(SPACE)-Wl,"$(RPATH_LINK)" -o "$(<)" $(START-GROUP) "$(>:T)" "$(LIBRARIES)" $(FINDLIBS-ST-PFX) -l$(FINDLIBS-ST) $(FINDLIBS-SA-PFX) -l$(FINDLIBS-SA) $(END-GROUP) $(OPTIONS) $(USER_OPTIONS)) + "$(CONFIG_COMMAND)" @($(<[1]:T).rsp:O=FC:<=@":>=":E=-L"$(LINKPATH)" -Wl,$(RPATH_OPTION)$(SPACE)-Wl,$(RPATH) -Wl,-rpath-link$(SPACE)-Wl,"$(RPATH_LINK)" -o "$(<)" $(START-GROUP) "$(>:T)" "$(LIBRARIES)" $(FINDLIBS-ST-PFX) -l$(FINDLIBS-ST) $(FINDLIBS-SA-PFX) -l$(FINDLIBS-SA) $(END-GROUP) $(OPTIONS) $(USER_OPTIONS)) } actions link.dll bind LIBRARIES { - "$(CONFIG_COMMAND)" @($(<[1]:T).rsp:O=FC:<=@":>=":E=-L"$(LINKPATH)" -Wl,$(RPATH_OPTION:E=-R)$(SPACE)-Wl,$(RPATH) -Wl,$(IMPLIB_OPTION:E=--out-implib),"$(<[2])" -o "$(<[1])" $(HAVE_SONAME)-Wl,$(SONAME_OPTION)$(SPACE)-Wl,"$(SONAME_PREFIX:E=)$(<[1]:D=)" $(SHARED_OPTION:E=-shared) $(START-GROUP) "$(>:T)" "$(LIBRARIES)" $(FINDLIBS-ST-PFX) -l$(FINDLIBS-ST) $(FINDLIBS-SA-PFX) -l$(FINDLIBS-SA) $(END-GROUP) $(OPTIONS) $(USER_OPTIONS)) + "$(CONFIG_COMMAND)" @($(<[1]:T).rsp:O=FC:<=@":>=":E=-L"$(LINKPATH)" -Wl,$(RPATH_OPTION)$(SPACE)-Wl,$(RPATH) -Wl,$(IMPLIB_OPTION:E=--out-implib),"$(<[2])" -o "$(<[1])" $(HAVE_SONAME)-Wl,$(SONAME_OPTION)$(SPACE)-Wl,"$(SONAME_PREFIX:E=)$(<[1]:D=)" $(SHARED_OPTION:E=-shared) $(START-GROUP) "$(>:T)" "$(LIBRARIES)" $(FINDLIBS-ST-PFX) -l$(FINDLIBS-ST) $(FINDLIBS-SA-PFX) -l$(FINDLIBS-SA) $(END-GROUP) $(OPTIONS) $(USER_OPTIONS)) } ### diff --git a/test/BoostBuild.py b/test/BoostBuild.py index bed89c8bfb..22ecccd067 100644 --- a/test/BoostBuild.py +++ b/test/BoostBuild.py @@ -113,14 +113,14 @@ def get_toolset(): # Detect the host OS. if sys.platform == "cygwin": - default_os = "cygwin" + host_os = "cygwin" elif sys.platform == "win32": - default_os = "windows" + host_os = "windows" elif hasattr(os, "uname"): - default_os = os.uname()[0].lower() + host_os = os.uname()[0].lower() -def expand_toolset(toolset, target_os=default_os): +def expand_toolset(toolset, target_os): match = re.match(r'^(clang|intel)(-[\d\.]+|)$', toolset) if match: if match.group(1) == "intel" and target_os == "windows": @@ -133,7 +133,7 @@ def expand_toolset(toolset, target_os=default_os): return toolset -def prepare_prefixes_and_suffixes(toolset, target_os=default_os): +def prepare_prefixes_and_suffixes(toolset, target_os): ind = toolset.find('-') if ind == -1: rtoolset = toolset @@ -143,7 +143,7 @@ def prepare_prefixes_and_suffixes(toolset, target_os=default_os): prepare_library_prefix(rtoolset, target_os) -def prepare_suffix_map(toolset, target_os=default_os): +def prepare_suffix_map(toolset, target_os): """ Set up suffix translation performed by the Boost Build testing framework to accommodate different toolsets generating targets of the same type using @@ -175,8 +175,12 @@ def prepare_suffix_map(toolset, target_os=default_os): if target_os == "darwin": suffixes[".dll"] = ".dylib" + if toolset == "emscripten": + suffixes[".exe"] = ".js" # or .wasm? + suffixes[".dll"] = ".so" # .wasn doesn't work for searched libs -def prepare_library_prefix(toolset, target_os=default_os): + +def prepare_library_prefix(toolset, target_os): """ Setup whether Boost Build is expected to automatically prepend prefixes to its built library targets. @@ -282,14 +286,9 @@ def __init__(self, arguments=None, executable=None, self.translate_suffixes = translate_suffixes self.use_test_config = use_test_config - self.target_os = default_os - self.toolset = get_toolset() - self.expanded_toolset = expand_toolset(self.toolset) - self.pass_toolset = pass_toolset + self.set_toolset(get_toolset(), _pass_toolset=pass_toolset) self.ignore_toolset_requirements = ignore_toolset_requirements - prepare_prefixes_and_suffixes(pass_toolset and self.toolset or "gcc") - use_default_bjam = "--default-bjam" in sys.argv if not use_default_bjam: @@ -343,12 +342,14 @@ def cleanup(self): # this case. pass - def set_toolset(self, toolset, target_os=default_os): - self.target_os = target_os - self.toolset = toolset - self.expanded_toolset = expand_toolset(toolset, target_os) - self.pass_toolset = True - prepare_prefixes_and_suffixes(toolset, target_os) + def set_toolset(self, toolset, target_os=None, _pass_toolset=True): + self.toolset = _pass_toolset and toolset or "gcc" + if not target_os and self.toolset.startswith("emscripten"): + target_os = "unknown" + self.target_os = target_os or host_os + self.expanded_toolset = expand_toolset(self.toolset, self.target_os) + self.pass_toolset = _pass_toolset + prepare_prefixes_and_suffixes(self.toolset, self.target_os) def is_implib_expected(self): return self.target_os in ["windows", "cygwin"] and not re.match(r'^clang(-linux)?(-[\d.]+)?$', self.toolset) @@ -756,6 +757,9 @@ def __ignore_junk(self): self.ignore("*.manifest") # MSVC DLL manifests. self.ignore("bin/standalone/msvc/*/msvc-setup.bat") + # emscripten 'exe' is .js which is a laucnher for .wasm file + self.ignore("*.wasm") + # Debug builds of bjam built with gcc produce this profiling data. self.ignore("gmon.out") self.ignore("*/gmon.out") diff --git a/test/conditionals.py b/test/conditionals.py index 208020fb5b..b8e9c6ed71 100644 --- a/test/conditionals.py +++ b/test/conditionals.py @@ -23,7 +23,7 @@ # Test conditionals in target requirements. t.write("jamroot.jam", "exe a : a.cpp : static:STATIC ;") t.run_build_system(["link=static"]) -t.expect_addition("bin/$toolset/debug/link-static*/a.exe") +t.expect_addition("bin/$toolset/debug*/a.exe") t.rm("bin") # Test conditionals in project requirements. @@ -32,7 +32,7 @@ exe a : a.cpp ; """) t.run_build_system(["link=static"]) -t.expect_addition("bin/$toolset/debug/link-static*/a.exe") +t.expect_addition("bin/$toolset/debug*/a.exe") t.rm("bin") # Regression test for a bug found by Ali Azarbayejani. Conditionals inside @@ -43,6 +43,6 @@ """) t.write("l.cpp", "int i;") t.run_build_system(["link=static"]) -t.expect_addition("bin/$toolset/debug/link-static*/a.exe") +t.expect_addition("bin/$toolset/debug*/a.exe") t.cleanup() diff --git a/test/unit_test.py b/test/unit_test.py index 8a070f7863..b9f4f57100 100644 --- a/test/unit_test.py +++ b/test/unit_test.py @@ -31,6 +31,6 @@ """) t.run_build_system(["link=static"]) -t.expect_addition("bin/$toolset/debug/link-static*/test.passed") +t.expect_addition("bin/$toolset/debug*/test.passed") t.cleanup()