Skip to content

Commit

Permalink
Bump Pyodide to v0.24.1
Browse files Browse the repository at this point in the history
includes a bump to Emscripten 3.1.45 and a workaround for a missing
file upstream
  • Loading branch information
tomjakubowski committed Nov 1, 2023
1 parent 785b1ee commit 2d5bbff
Show file tree
Hide file tree
Showing 5 changed files with 130 additions and 14 deletions.
17 changes: 14 additions & 3 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1126,13 +1126,14 @@ jobs:
runs-on: ${{ matrix.os }}
needs: [initialize]
env:
PYODIDE_VERSION: 0.23.2
# NOTA BENE: When upgrading past 0.24.1, remove workaround below for https://github.com/pyodide/pyodide/issues/4216
PYODIDE_VERSION: 0.24.1
# PYTHON_VERSION and EMSCRIPTEN_VERSION are determined by PYODIDE_VERSION.
# The appropriate versions can be found in the Pyodide repodata.json
# "info" field, or in Makefile.envs:
# https://github.com/pyodide/pyodide/blob/main/Makefile.envs#L2
PYTHON_VERSION: 3.11.2
EMSCRIPTEN_VERSION: 3.1.32
PYTHON_VERSION: 3.11.3
EMSCRIPTEN_VERSION: 3.1.45
steps:
- name: Checkout perspective
uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
Expand Down Expand Up @@ -1160,6 +1161,16 @@ jobs:
- name: Install pyodide-build
run: pip install pyodide-build==$PYODIDE_VERSION "pydantic<2"

# NOTA BENE: When removing this step, also remove the vendored pyodide file
- name: Provision workaround for bug in Pyodide 0.24.1
run: |
PYTHON_HOME="$(python -c 'import sys; print(sys.prefix)')"
TOOLCHAIN_DIR="$PYTHON_HOME/lib/python3.11/site-packages/pyodide_build/tools/cmake/Modules/Platform"
echo "$PYTHON_HOME"
echo "$TOOLCHAIN_DIR"
mkdir -p "$TOOLCHAIN_DIR"
cp vendor/pyodide/Emscripten.cmake "$TOOLCHAIN_DIR"
- name: Build
# Without --exports=pyinit, pyodide-build tries to export all symbols from every .o,
# which causes the em++ linker command line invocation to be so long
Expand Down
4 changes: 1 addition & 3 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,6 @@ auto-save-list
tramp
.\#*

# npm lock file
package-lock.json

# Intellij/WebStorm IDE
*.iml
/.idea
Expand Down Expand Up @@ -114,6 +111,7 @@ tools/perspective-build/lib
packages/perspective-esbuild-plugin/lib
python/perspective/perspective/nbextension/static
python/perspective/perspective/labextension
boost_*.tar.gz

# editor, IDE, OS
.DS_Store
Expand Down
14 changes: 7 additions & 7 deletions cmake/modules/FindInstallDependency.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -34,14 +34,14 @@ function(psp_build_dep name cmake_file)
${CMAKE_BINARY_DIR}/${name}-build
EXCLUDE_FROM_ALL)

include_directories(${CMAKE_BINARY_DIR}/${name}-src/cpp/src/)
include_directories(SYSTEM ${CMAKE_BINARY_DIR}/${name}-src/cpp/src/)
elseif(${name} STREQUAL exprtk)
# no cmakelists - just include the header
include_directories(${CMAKE_BINARY_DIR}/${name}-src)
include_directories(SYSTEM ${CMAKE_BINARY_DIR}/${name}-src)
elseif(${name} STREQUAL re2)
# Overwrite re2's CMakeLists with our custom CMakeLists.
configure_file(${PSP_CMAKE_MODULE_PATH}/${name}/CMakeLists.txt ${CMAKE_BINARY_DIR}/${name}-src/ COPYONLY)
include_directories(${CMAKE_BINARY_DIR}/${name}-src)
include_directories(SYSTEM ${CMAKE_BINARY_DIR}/${name}-src)

add_subdirectory(${CMAKE_BINARY_DIR}/${name}-src
${CMAKE_BINARY_DIR}/${name}-build
Expand All @@ -51,15 +51,15 @@ function(psp_build_dep name cmake_file)
add_subdirectory(${CMAKE_BINARY_DIR}/${name}-src/build/cmake
${CMAKE_BINARY_DIR}/${name}-build
EXCLUDE_FROM_ALL)
include_directories(${CMAKE_BINARY_DIR}/${name}-src/lib)
include_directories(SYSTEM ${CMAKE_BINARY_DIR}/${name}-src/lib)
else()
add_subdirectory(${CMAKE_BINARY_DIR}/${name}-src
${CMAKE_BINARY_DIR}/${name}-build
EXCLUDE_FROM_ALL)

include_directories(${CMAKE_BINARY_DIR}/${name}-src/extras/${name}/include)
include_directories(${CMAKE_BINARY_DIR}/${name}-src/include)
include_directories(${CMAKE_BINARY_DIR}/${name}-src)
include_directories(SYSTEM ${CMAKE_BINARY_DIR}/${name}-src/extras/${name}/include)
include_directories(SYSTEM ${CMAKE_BINARY_DIR}/${name}-src/include)
include_directories(SYSTEM ${CMAKE_BINARY_DIR}/${name}-src)
endif()

if(NOT PSP_WASM_BUILD AND (MACOS OR NOT MANYLINUX))
Expand Down
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
"internal": "Internal"
}
},
"emscripten": "3.1.32",
"emscripten": "3.1.45",
"engines": {
"node": ">=14.18.2"
},
Expand Down
107 changes: 107 additions & 0 deletions vendor/pyodide/Emscripten.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
###############################################################################################
# This file is a 'toolchain description file' for CMake. #
# Differences from the original Emscripten toolchain file are: #
# - set TARGET_SUPPORTS_SHARED_LIBS to TRUE #
# - set CMAKE_INSTALL_PREFIX to WASM_LIBRARY_DIR #
# - disable the usage of response file for object and libraries #
# - allow to overwrite CMAKE_PROJECT_INCLUDE, CMAKE_PROJECT_INCLUDE_BEFORE with env variable #
# - append SIDE_MODULE_CFLAGS and SIDE_MODULE_LDFLAGS automatically #
###############################################################################################

# First, we inherit most of settings from the original Emscripten toolchain file.
# Then, we will update some settings.

execute_process(COMMAND "em-config" "EMSCRIPTEN_ROOT"
RESULT_VARIABLE _emconfig_result
OUTPUT_VARIABLE _emconfig_output
OUTPUT_STRIP_TRAILING_WHITESPACE)
if (NOT _emconfig_result EQUAL 0)
message(FATAL_ERROR "Failed to find emscripten root directory with command \"em-config EMSCRIPTEN_ROOT\"! Process returned with error code ${_emcache_result}.")
endif()

file(TO_CMAKE_PATH "${_emconfig_output}" _emcache_output)
set(EMSCRIPTEN_CMAKE_TOOLCHAIN_FILE "${_emconfig_output}/cmake/Modules/Platform/Emscripten.cmake" CACHE FILEPATH "Path to Emscripten CMake toolchain file.")
include("${EMSCRIPTEN_CMAKE_TOOLCHAIN_FILE}")

# From Emscripten 3.1.33, CACHE property was removed from these variables,
# so it is not possible to overwrite them by -D<var>=<value>.
# See: https://github.com/emscripten-core/emscripten/pull/18764
# But we need to do it, so set them again here...
unset(CMAKE_C_COMPILER)
unset(CMAKE_CXX_COMPILER)
unset(CMAKE_AR)
unset(CMAKE_RANLIB)
unset(CMAKE_C_COMPILER_AR)
unset(CMAKE_CXX_COMPILER_AR)
unset(CMAKE_C_COMPILER_RANLIB)
unset(CMAKE_CXX_COMPILER_RANLIB)

set(CMAKE_C_COMPILER "${EMSCRIPTEN_ROOT_PATH}/emcc${EMCC_SUFFIX}" CACHE FILEPATH "Emscripten emcc")
set(CMAKE_CXX_COMPILER "${EMSCRIPTEN_ROOT_PATH}/em++${EMCC_SUFFIX}" CACHE FILEPATH "Emscripten em++")
set(CMAKE_AR "${EMSCRIPTEN_ROOT_PATH}/emar${EMCC_SUFFIX}" CACHE FILEPATH "Emscripten ar")
set(CMAKE_RANLIB "${EMSCRIPTEN_ROOT_PATH}/emranlib${EMCC_SUFFIX}" CACHE FILEPATH "Emscripten ranlib")
set(CMAKE_C_COMPILER_AR "${CMAKE_AR}" CACHE FILEPATH "Emscripten ar")
set(CMAKE_CXX_COMPILER_AR "${CMAKE_AR}" CACHE FILEPATH "Emscripten ar")
set(CMAKE_C_COMPILER_RANLIB "${CMAKE_RANLIB}" CACHE FILEPATH "Emscripten ranlib")
set(CMAKE_CXX_COMPILER_RANLIB "${CMAKE_RANLIB}" CACHE FILEPATH "Emscripten ranlib")

# Note: this is False in original Emscripten toolchain,
# however we always want to allow build shared libs
# (See also: https://github.com/emscripten-core/emscripten/pull/16281)
set_property(GLOBAL PROPERTY TARGET_SUPPORTS_SHARED_LIBS TRUE)

# CMakeSystemSpecificInformation.cmake tries to include a Toolchain file from ${CMAKE_MODULE_PATH}/Platform/{CMAKE_SYSTEM_NAME}.cmake.
# So in order to prevent CMake from loading the original Emscripten toolchain file, we need to prepend our own toolchain file.
list(PREPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/..")

# We build libraries into WASM_LIBRARY_DIR, so lets tell CMake
# to find libraries from there.
if (NOT "$ENV{WASM_LIBRARY_DIR}" STREQUAL "")
list(PREPEND CMAKE_FIND_ROOT_PATH "$ENV{WASM_LIBRARY_DIR}")
if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
set(CMAKE_INSTALL_PREFIX "$ENV{WASM_LIBRARY_DIR}" CACHE PATH
"Install path prefix, prepended onto install directories." FORCE)
endif()
endif()

# Note: Emscripten installs libraries into subdirectories such as:
# - Non PIC: <SYSROOT>/lib/<arch>/<lib>
# - PIC: <SYSROOT>/lib/<arch>/pic/<lib>
# - LTO: <SYSROOT>/lib/<arch>/lto/<lib>
# - PIC+LTO: <SYSROOT>/lib/<arch>/pic/lto/<lib>
# We always wants to use a library built with "-fPIC", but
# CMake's find_library() will search Non-PIC dir only by default.
# This is a hack which overrides find_library() to tell CMake to look at PIC dirs first.
if ($ENV{CFLAGS} MATCHES "MEMORY64")
set(CMAKE_LIBRARY_ARCHITECTURE "wasm64-emscripten/pic")
else()
set(CMAKE_LIBRARY_ARCHITECTURE "wasm32-emscripten/pic")
endif()

set(CMAKE_C_USE_RESPONSE_FILE_FOR_LIBRARIES 0)
set(CMAKE_CXX_USE_RESPONSE_FILE_FOR_LIBRARIES 0)
set(CMAKE_C_USE_RESPONSE_FILE_FOR_OBJECTS 1)
set(CMAKE_CXX_USE_RESPONSE_FILE_FOR_OBJECTS 1)
set(CMAKE_C_USE_RESPONSE_FILE_FOR_INCLUDES 1)
set(CMAKE_CXX_USE_RESPONSE_FILE_FOR_INCLUDES 1)

# Allow some of the variables to be overridden by the user by env variable
# CMAKE_PROJECT_INCLUDE is a nice way to override some properties of the project
# It can be set by passing -DCMAKE_PROJECT_INCLUDE=... to cmake
# But some python packages will not allow users to pass extra arguments to cmake
# So this is a workaround to allow users to override CMAKE_PROJECT_INCLUDE by env variable
if ("${CMAKE_PROJECT_INCLUDE_BEFORE}" STREQUAL "" AND DEFINED ENV{CMAKE_PROJECT_INCLUDE_BEFORE})
message(STATUS "Set CMAKE_PROJECT_INCLUDE_BEFORE to $ENV{CMAKE_PROJECT_INCLUDE_BEFORE} using env variable")
set(CMAKE_PROJECT_INCLUDE_BEFORE "$ENV{CMAKE_PROJECT_INCLUDE_BEFORE}")
endif()

if ("${CMAKE_PROJECT_INCLUDE}" STREQUAL "" AND DEFINED ENV{CMAKE_PROJECT_INCLUDE})
message(STATUS "Set CMAKE_PROJECT_INCLUDE to $ENV{CMAKE_PROJECT_INCLUDE} using env variable")
set(CMAKE_PROJECT_INCLUDE "$ENV{CMAKE_PROJECT_INCLUDE}")
endif()

# Set SIDE_MODULE_CFLAGS and SIDE_MODULE_LDFLAGS automatically
# This is automatically set for Python project, but we often omit these in libraries.
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} $ENV{SIDE_MODULE_CFLAGS}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} $ENV{SIDE_MODULE_CXXFLAGS}")
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} $ENV{SIDE_MODULE_LDFLAGS}")

0 comments on commit 2d5bbff

Please sign in to comment.