diff --git a/.github/matrix.json b/.github/matrix.json index c319f1899..c32dc4d71 100644 --- a/.github/matrix.json +++ b/.github/matrix.json @@ -24,6 +24,11 @@ "version": "16", "os": "windows-2019", "cmake_toolset": "Visual Studio 16 2019" + }, + { + "name": "Linux Webassembly", + "compiler": "emscripten", + "os": "ubuntu-22.04" } ] } diff --git a/.github/workflows/stlab.yml b/.github/workflows/stlab.yml index 9773e91af..7f7712ca9 100644 --- a/.github/workflows/stlab.yml +++ b/.github/workflows/stlab.yml @@ -19,7 +19,7 @@ jobs: id: set-matrix # Note: The json in this variable must be a single line for parsing to succeed. run: echo "::set-output name=matrix::$(cat .github/matrix.json | scripts/flatten_json.py)" - + builds: needs: generate-matrix runs-on: ${{ matrix.config.os }} @@ -31,63 +31,106 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Install dependencies (macos) + - name: Install dependencies // macOS if: ${{ startsWith(matrix.config.os, 'macos') }} run: | + brew update brew install boost brew install ninja shell: bash - - name: Install dependencies (ubuntu) - if: ${{ startsWith(matrix.config.os, 'ubuntu') }} + - name: Install dependencies // Linux (GCC|Clang) + if: ${{ startsWith(matrix.config.os, 'ubuntu') && !startsWith(matrix.config.compiler, 'emscripten') }} run: | + sudo apt-get update sudo apt-get install -y ninja-build sudo apt-get install -y libboost-all-dev shell: bash - - name: Install dependencies (Windows) + - name: Install dependencies // Windows if: ${{ startsWith(matrix.config.os, 'windows') }} run: | choco install --yes ninja vcpkg install boost-test:x64-windows boost-multiprecision:x64-windows boost-variant:x64-windows shell: cmd - - name: Set enviroment variables (Linux+GCC) + - name: Install dependencies // Linux Emscripten + if: ${{ startsWith(matrix.config.compiler, 'emscripten') }} + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y ninja-build + git clone --depth 1 --recurse-submodules --shallow-submodules --jobs=8 https://github.com/boostorg/boost.git $HOME/boost + + git clone --depth 1 https://github.com/emscripten-core/emsdk.git $HOME/emsdk + pushd $HOME/emsdk + ./emsdk install latest + ./emsdk activate latest + echo 'source "$HOME/emsdk/emsdk_env.sh"' >> $HOME/.bash_profile + + # Override Emsdk's bundled node (14.18.2) to the GH Actions system installation (>= 16.16.0) + sed -i "/^NODE_JS = .*/c\NODE_JS = '`which node`'" .emscripten + echo "Overwrote .emscripten config file to:" + cat .emscripten + popd + + - name: Set enviroment variables // Linux GCC if: ${{ matrix.config.compiler == 'gcc' }} shell: bash run: | echo "CC=gcc-${{matrix.config.version}}" >> $GITHUB_ENV echo "CXX=g++-${{matrix.config.version}}" >> $GITHUB_ENV - - name: Set enviroment variables (Linux+Clang) + - name: Set enviroment variables // Linux Clang if: ${{ matrix.config.compiler == 'clang' }} shell: bash run: | echo "CC=clang-${{matrix.config.version}}" >> $GITHUB_ENV echo "CXX=clang++-${{matrix.config.version}}" >> $GITHUB_ENV - - name: Configure (Unix) - if: ${{ startsWith(matrix.config.os, 'ubuntu') || startsWith(matrix.config.os, 'macos') }} + - name: Compile Boost // Emscripten + if: ${{ startsWith(matrix.config.compiler, 'emscripten') }} + shell: bash -l {0} + run: | + mkdir -p ../build-boost + cmake -S $HOME/boost -B ../build-boost -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=23 \ + -DCMAKE_CXX_FLAGS="-Wno-deprecated-builtins" \ + -DCMAKE_TOOLCHAIN_FILE=$GITHUB_WORKSPACE/cmake/Platform/Emscripten-STLab.cmake \ + -DBOOST_INCLUDE_LIBRARIES="optional;variant;multiprecision;test" + + cmake --build ../build-boost + cmake --install ../build-boost + + - name: Configure // Unix !Emscripten + if: ${{ (startsWith(matrix.config.os, 'ubuntu') || startsWith(matrix.config.os, 'macos')) && !startsWith(matrix.config.compiler, 'emscripten') }} shell: bash run: | mkdir ../build cmake -S. -B../build -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=23 - - name: Configure (Windows) + - name: Configure // Linux Emscripten + if: ${{ startsWith(matrix.config.compiler, 'emscripten') }} + shell: bash -l {0} + run: | + mkdir ../build + cmake -S. -B../build -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=23 \ + -DCMAKE_TOOLCHAIN_FILE=$GITHUB_WORKSPACE/cmake/Platform/Emscripten-STLab.cmake + + - name: Configure // Windows if: ${{ startsWith(matrix.config.os, 'windows') }} shell: cmd run: | call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 mkdir ..\build - cmake -S. -B../build -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_CXX_STANDARD=23 + cmake -S. -B../build -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=23 -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake - - name: Build (Unix) + - name: Build // Unix if: ${{ startsWith(matrix.config.os, 'ubuntu') || startsWith(matrix.config.os, 'macos') }} shell: bash run: | cmake --build ../build/ - - name: Build (windows) + - name: Build // Windows if: ${{ startsWith(matrix.config.os, 'windows') }} shell: cmd run: | @@ -98,4 +141,4 @@ jobs: shell: bash run: | cd ../build/ - ctest + ctest --output-on-failure diff --git a/cmake/Platform/Emscripten-STLab.cmake b/cmake/Platform/Emscripten-STLab.cmake new file mode 100644 index 000000000..0ebc1af51 --- /dev/null +++ b/cmake/Platform/Emscripten-STLab.cmake @@ -0,0 +1,139 @@ +# +# This toolchain file extends `Emscripten.cmake` provided by the Emscripten SDK, +# and set options required to run STLab test drivers with +# CTest (using a node runner). +# + +# +# Find the Emscripten SDK and include its CMake toolchain. +# +find_program( EM_CONFIG_EXECUTABLE em-config ) +if ( NOT EM_CONFIG_EXECUTABLE ) + message( FATAL_ERROR "Could not find emsdk installation. Please install the Emscripten SDK.\nhttps://emscripten.org/docs/getting_started/downloads.html" ) +endif() + +execute_process( COMMAND ${EM_CONFIG_EXECUTABLE} EMSCRIPTEN_ROOT OUTPUT_VARIABLE EMSDK_ROOT OUTPUT_STRIP_TRAILING_WHITESPACE ) +include( ${EMSDK_ROOT}/cmake/Modules/Platform/Emscripten.cmake ) + +# +# Set compiler and linker flags. +# + +# +# `-pthread` +# STLab uses threads. Without these, the tests will not compile. +# +set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread" ) +set( CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -pthread" ) + +# +# `-fwasm-exceptions`: +# STLab uses exceptions. Without these, the tests error out with: +# +# Pthread 0x005141d0 sent an error! http://localhost:6931/: uncaught exception: 10570976 \ +# - Exception catching is disabled, this exception cannot be caught. +# +set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fwasm-exceptions" ) +set( CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fwasm-exceptions" ) + +# +# `-sSUPPORT_LONGJMP=wasm` +# Enables experimental support for LONGJMP in functions which may throw exceptions. +# Without this, Boost doesn't compile (LLVM errors out). +# +set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -sSUPPORT_LONGJMP=wasm" ) + +# +# `-sEXIT_RUNTIME=1` +# Indicates the runtime environment (node) should exit when `main()` returns. +# +set( CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -sEXIT_RUNTIME=1" ) + +# +# `-sINITIAL_MEMORY=300MB` +# Without this, the tests throw an Out Of Memory error (OOM). The first sign is an error out with the following: +# +# Pthread 0x0058faf8 sent an error! http://localhost:6931/: RuntimeError: unreachable executed +# +# If the problematic test is run in a browser with `emrun`, JavaScript errors are emitted that explain: +# +# Aborted(Cannot enlarge memory arrays to size 17457152 bytes (OOM). Either +# (1) compile with -sINITIAL_MEMORY=X with X higher than the current value 16777216, +# (2) compile with -sALLOW_MEMORY_GROWTH which allows increasing the size at runtime, or +# (3) if you want malloc to return NULL (0) instead of this abort, compile with -sABORTING_MALLOC=0) +# +# Note that (2) is not an option because pthread cannot yet be combined with -sALLOW_MEMORY_GROWTH: +# See https://github.com/WebAssembly/design/issues/1271 +# Smaller values (150MB, 200MB) produce intermittent failures. 300MB was chosen to give enough headroom for +# tests written in the future. +# +set( CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -sINITIAL_MEMORY=300MB" ) + +# +# `-sPTHREAD_POOL_SIZE=32` +# Without this, the tests deadlock. Lower values were tested. +# 8 threads deadlocked consistently, 16 threads passed consistently. +# 32 was chosen to give enough headroom for tests written in the future. +# +set( CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -sPTHREAD_POOL_SIZE=32" ) + +# +# `-sPROXY_TO_PTHREAD` +# This flag wraps our executable's main function in a pthread. +# Without this, we exhaust the thread pool very quickly. The error looks like this: +# +# Tried to spawn a new thread, but the thread pool is exhausted. +# This might result in a deadlock unless some threads eventually exit or the code explicitly breaks out to the event loop. +# +# You can read more about the setting here: https://emscripten.org/docs/porting/pthreads.html#blocking-on-the-main-browser-thread +# +set( CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -sPROXY_TO_PTHREAD" ) + +# +# Set the minimum required version for node; earlier versions lack sufficient exception support. +# Note: https://www.npmjs.com/package/wasm-check is a useful utility to find which +# --experimental-wasm-xxx flags are supported by node. +# +set( STLAB_WASM_NODE_JS_MIN_VERSION "16.16.0" ) + +set( NODE_JS_FLAGS "--experimental-wasm-threads;--experimental-wasm-eh" ) + +# +# Check if NODE_JS_EXECUTABLE (found by find_program() in Emscripten.cmake) is recent enough for STLab. +# Set CMAKE_CROSSCOMPILING_EMULATOR to a sufficiently recent node + required experimental flags. +# +if ( NOT NODE_JS_EXECUTABLE ) + message( FATAL_ERROR "stlab:wasm: Unable to find node. Please install ${STLAB_WASM_NODE_JS_MIN_VERSION} or newer." ) +endif() + +message( STATUS "stlab:wasm: Ensuring ${NODE_JS_EXECUTABLE} is at least ${STLAB_WASM_NODE_JS_MIN_VERSION}..." ) +execute_process( COMMAND ${NODE_JS_EXECUTABLE} --version OUTPUT_VARIABLE NODE_JS_EXECUTABLE_V_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE ) +STRING( REPLACE "v" "" NODE_JS_EXECUTABLE_VERSION ${NODE_JS_EXECUTABLE_V_VERSION} ) + +if ( NODE_JS_EXECUTABLE_VERSION VERSION_LESS ${STLAB_WASM_NODE_JS_MIN_VERSION} ) + message( FATAL_ERROR "stlab:wasm: Unsupported node: ${NODE_JS_EXECUTABLE_VERSION}. Please install ${STLAB_WASM_NODE_JS_MIN_VERSION} or newer." ) +endif() + +message( STATUS "stlab:wasm: Installed node satisfies requirements: ${NODE_JS_EXECUTABLE_VERSION}" ) +set( CMAKE_CROSSCOMPILING_EMULATOR "${NODE_JS_EXECUTABLE};${NODE_JS_FLAGS}" ) + +# +# Emscripten supports dynamic linking, but doing so introduces some complexity: +# https://emscripten.org/docs/compiling/Dynamic-Linking.html +# We presently have no need to dynamically link WASM modules, so we instruct +# boost to link statically. +# +# It would be nice if Boost respected (BUILD_SHARED_LIBS OFF), but it does not. +# +set( BUILD_SHARED_LIBS OFF ) +set( Boost_USE_STATIC_LIBS ON ) + +# +# Print the emcc version information, if relevant. +# +execute_process( COMMAND emcc -v ERROR_VARIABLE EMCC_VERSION ) +STRING( REGEX REPLACE "\n" ";" EMCC_VERSION "${EMCC_VERSION}" ) +message ( STATUS "stlab: Emscripten version:" ) +foreach( LINE ${EMCC_VERSION} ) + message ( STATUS "\t${LINE}" ) +endforeach() diff --git a/stlab/concurrency/main_executor.hpp b/stlab/concurrency/main_executor.hpp index 5a47c68e7..fa5bb1d74 100644 --- a/stlab/concurrency/main_executor.hpp +++ b/stlab/concurrency/main_executor.hpp @@ -104,7 +104,46 @@ struct main_executor_type { #elif STLAB_MAIN_EXECUTOR(EMSCRIPTEN) -using main_executor_type = default_executor_type; +struct main_scheduler_type { + using result_type = void; + + template + void operator()(F&& f) const { + using function_type = typename std::remove_reference::type; + auto p = new function_type(std::forward(f)); + + /* + `emscripten_async_run_in_main_runtime_thread()` schedules a function to run on the main + JS thread, however, the code can be executed at any POSIX thread cancelation point if + wasm code is executing on the JS main thread. + Executing the code from a POSIX thread cancelation point can cause problems, including + deadlocks and data corruption. Consider: + ``` + mutex.lock(); // <-- If reentered, would deadlock here + new T; // <-- POSIX cancelation point, could reenter + ``` + The call to `emscripten_async_call()` bounces the call to execute as part of the main + run-loop on the current (main) thread. This avoids nasty reentrancy issues if executed + from a POSIX thread cancelation point. + */ + + emscripten_async_run_in_main_runtime_thread( + EM_FUNC_SIG_VI, + static_cast([](void* f_) { + emscripten_async_call( + [](void* f_) { + auto f = static_cast(f_); + // Note the absence of exception handling. + // Operations queued to the task system cannot throw as a precondition. + // We use packaged tasks to marshal exceptions. + (*f)(); + delete f; + }, + f_, 0); + }), + p); + } +}; #elif STLAB_MAIN_EXECUTOR(NONE)