diff --git a/.circleci/config.yml b/.circleci/config.yml index 0e2cb2c4e9bda..5b8005721c870 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -77,6 +77,7 @@ commands: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y export PATH=${HOME}/.cargo/bin:${PATH} rustup target add wasm32-unknown-emscripten + cargo install wasm-bindgen-cli echo "export PATH=\"\$HOME/.cargo/bin:\$PATH\"" >> $BASH_ENV install-node-version: description: "install a specific version of node" diff --git a/emcc.py b/emcc.py index f770f4a539b06..60dda0a305644 100644 --- a/emcc.py +++ b/emcc.py @@ -27,7 +27,6 @@ import shutil import sys import tarfile -from dataclasses import dataclass from enum import Enum, auto, unique # This assert needs to happen early, before any too-recent python syntax is used. @@ -51,7 +50,7 @@ from tools.settings import COMPILE_TIME_SETTINGS, default_setting, settings, user_settings from tools.shared import DEBUG, DYLIB_EXTENSIONS, in_temp from tools.toolchain_profiler import ToolchainProfiler -from tools.utils import exit_with_error, get_file_suffix, read_file, unsuffixed_basename +from tools.utils import LinkFlag, exit_with_error, get_file_suffix, read_file, unsuffixed_basename logger = logging.getLogger('emcc') @@ -96,20 +95,6 @@ class Mode(Enum): COMPILE_AND_LINK = auto() -@dataclass -class LinkFlag: - """Used to represent a linker flag. - - The flag value is stored along with a bool that distinguishes input - files from non-files. - - A list of these is returned by separate_linker_flags. - """ - - value: str - is_file: int - - class EmccState: def __init__(self, args): self.mode = Mode.COMPILE_AND_LINK @@ -578,7 +563,7 @@ def compile_source_file(input_file): # Default to assuming the inputs are object files and pass them to the linker pass - return [f.value for f in linker_args] + return linker_args if __name__ == '__main__': diff --git a/site/source/docs/tools_reference/settings_reference.rst b/site/source/docs/tools_reference/settings_reference.rst index c61cd19e1dc15..539d4aa4a245a 100644 --- a/site/source/docs/tools_reference/settings_reference.rst +++ b/site/source/docs/tools_reference/settings_reference.rst @@ -3286,6 +3286,15 @@ Example use ``-sSIGNATURE_CONVERSIONS=someFunction:_p,anotherFunction:p`` Default value: [] +.. _wasm_bindgen: + +WASM_BINDGEN +============ + +Run wasm-bindgen and integrate the rust-exported symbols into the rest of Emscripten's JS output. + +Default value: 0 + .. _source_phase_imports: SOURCE_PHASE_IMPORTS diff --git a/src/runtime_common.js b/src/runtime_common.js index e4818c0dc13bc..184755e17046d 100644 --- a/src/runtime_common.js +++ b/src/runtime_common.js @@ -131,7 +131,7 @@ function updateMemoryViews() { {{{ maybeExportHeap('HEAPU64') }}}HEAPU64 = new BigUint64Array(b); #endif #if SUPPORT_BIG_ENDIAN - {{{ maybeExportHeap('HEAP_DATA_VIEW') }}} HEAP_DATA_VIEW = new DataView(b); + {{{ maybeExportHeap('HEAP_DATA_VIEW') }}}HEAP_DATA_VIEW = new DataView(b); LE_HEAP_UPDATE(); #endif } diff --git a/src/settings.js b/src/settings.js index c9e8ad2f6eed5..21120d94d5f28 100644 --- a/src/settings.js +++ b/src/settings.js @@ -2166,6 +2166,10 @@ var LEGACY_RUNTIME = false; // [link] var SIGNATURE_CONVERSIONS = []; +// Run wasm-bindgen and integrate the rust-exported symbols into the rest of Emscripten's JS output. +// [link] +var WASM_BINDGEN = 0; + // Experimental support for wasm source phase imports. // This is only currently implemented in the pre-release/nightly version of // node, and not yet supported by browsers. diff --git a/test/common.py b/test/common.py index 758ba83c65b30..94d756229181f 100644 --- a/test/common.py +++ b/test/common.py @@ -1377,7 +1377,7 @@ def ccshared(src, linkto=None): return 0; } ''' % locals(), - 'a: loaded\na: b (prev: (null))\na: c (prev: b)\n', cflags=extra_args) + 'a: loaded\na: b (prev: (null))\na: c (prev: b)\n', cflags=extra_args) def do_run(self, src, expected_output=None, force_c=False, **kwargs): if 'no_build' in kwargs: diff --git a/test/rust/basics/.cargo/config.toml b/test/rust/basics/.cargo/config.toml new file mode 100644 index 0000000000000..3301c205541b8 --- /dev/null +++ b/test/rust/basics/.cargo/config.toml @@ -0,0 +1,2 @@ +[build] +target = "wasm32-unknown-emscripten" diff --git a/test/rust/bindgen_integration/.cargo/config.toml b/test/rust/bindgen_integration/.cargo/config.toml new file mode 100644 index 0000000000000..c9be0d51ea09a --- /dev/null +++ b/test/rust/bindgen_integration/.cargo/config.toml @@ -0,0 +1,7 @@ +[build] +target = "wasm32-unknown-emscripten" +rustflags = [ + "-Cllvm-args=-enable-emscripten-cxx-exceptions=0", + "-Cpanic=abort", + "-Crelocation-model=static", +] diff --git a/test/rust/bindgen_integration/Cargo.toml b/test/rust/bindgen_integration/Cargo.toml new file mode 100644 index 0000000000000..044293686a9e6 --- /dev/null +++ b/test/rust/bindgen_integration/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "bindgen_integration" +edition = "2021" + +[lib] +crate-type = ["staticlib"] + +[dependencies] diff --git a/test/rust/bindgen_integration/src/lib.rs b/test/rust/bindgen_integration/src/lib.rs new file mode 100644 index 0000000000000..541bc882468cb --- /dev/null +++ b/test/rust/bindgen_integration/src/lib.rs @@ -0,0 +1,6 @@ +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +pub fn rs_add(a: i32, b: i32) -> i32 { + return a + b; +} diff --git a/test/test_other.py b/test/test_other.py index 0d924f90e18a6..c8ba8492d6aa2 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -14806,6 +14806,22 @@ def test_rust_integration_basics(self): }''') self.do_runf('main.cpp', 'Hello from rust!', cflags=[lib]) + @requires_rust + def test_wasm_bindgen_integration(self): + copytree(test_file('rust/bindgen_integration'), '.') + self.run_process(['cargo', 'add', 'wasm-bindgen']) + self.run_process(['cargo', 'build']) + lib = 'target/wasm32-unknown-emscripten/debug/libbindgen_integration.a' + self.assertExists(lib) + + create_file('main.cpp', '') + create_file('post.js', ''' + Module.onRuntimeInitialized = () => out(Module.rs_add(17, 25)); + ''') + emcc_args = [lib, '-sWASM_BINDGEN', '--post-js=post.js', '-lexports.js'] + + self.do_runf('main.cpp', '42', cflags=emcc_args) + def test_relative_em_cache(self): with env_modify({'EM_CACHE': 'foo'}): self.assert_fail([EMCC, '-c', test_file('hello_world.c')], 'emcc: error: environment variable EM_CACHE must be an absolute path: foo') diff --git a/tools/building.py b/tools/building.py index 3cb7ba014c4be..cd38e7a0c7c0f 100644 --- a/tools/building.py +++ b/tools/building.py @@ -277,12 +277,33 @@ def lld_flags_for_executable(external_symbols): return cmd -def lld_flags(args): +def get_wasm_bindgen_exported_symbols(input_files): + nm_args = [LLVM_NM, '--defined-only', '--extern-only', '--format=just-symbols', + '--print-file-name', '--quiet'] + nm_args += input_files + + result = check_call(nm_args, stdout=PIPE) + symbols = [] + for line in result.stdout.splitlines(): + _, symbol = line.split() + # Skip mangled (non-C) symbols + if symbol.startswith(('_Z', '_R', 'anon.')): + continue + symbols.append(symbol) + + return symbols + + +def lld_flags(args, linker_inputs=None): # lld doesn't currently support --start-group/--end-group since the # semantics are more like the windows linker where there is no need for # grouping. args = [a for a in args if a not in {'--start-group', '--end-group'}] + if settings.WASM_BINDGEN: + exported_symbols = get_wasm_bindgen_exported_symbols(linker_inputs) + args.extend(f'--export-if-defined={e}' for e in exported_symbols) + # Emscripten currently expects linkable output (SIDE_MODULE/MAIN_MODULE) to # include all archive contents. if settings.LINKABLE and (settings.FAKE_DYLIBS or not settings.SIDE_MODULE): @@ -316,7 +337,7 @@ def lld_flags(args): return args -def link_lld(args, target, external_symbols=None): +def link_lld(args, target, external_symbols=None, linker_inputs=None): # runs lld to link things. if not os.path.exists(WASM_LD): exit_with_error('linker binary not found in LLVM directory: %s', WASM_LD) @@ -325,7 +346,7 @@ def link_lld(args, target, external_symbols=None): # normal linker flags that are used when building and executable if '--relocatable' not in args and '-r' not in args: cmd += lld_flags_for_executable(external_symbols) - cmd += lld_flags(args) + cmd += lld_flags(args, linker_inputs) cmd = get_command_with_possible_response_file(cmd) check_call(cmd) @@ -1256,6 +1277,34 @@ def run_wasm_opt(infile, outfile=None, args=[], **kwargs): # noqa return run_binaryen_command('wasm-opt', infile, outfile, args=args, **kwargs) +def run_wasm_bindgen(infile, outfile=None, args=[], **kwargs): # noqa + bindgen_out_dir = os.path.join(get_emscripten_temp_dir(), 'bindgen_out') + + wasm_bindgen_bin = shutil.which('wasm-bindgen') + if not wasm_bindgen_bin: + exit_with_error('wasm-bindgen executable not found in $PATH') + cmd = [ + wasm_bindgen_bin, + infile, + '--keep-lld-exports', + '--keep-debug', + '--out-dir', + bindgen_out_dir, + ] + check_call(cmd) + + # Don't try to predict the .wasm filename that wasm-bindgen outputs. Instead + # just grab the .wasm file itself. + all_output_files = os.listdir(bindgen_out_dir) + new_wasm_file = [x for x in all_output_files if x.endswith('.wasm')][0] + if outfile is None: + outfile = infile + + shutil.copyfile(os.path.join(bindgen_out_dir, new_wasm_file), outfile) + + return os.path.join(bindgen_out_dir, 'library_bindgen.js') + + intermediate_counter = 0 diff --git a/tools/emscripten.py b/tools/emscripten.py index 03686f60ab903..57a24f01d837a 100644 --- a/tools/emscripten.py +++ b/tools/emscripten.py @@ -680,7 +680,7 @@ def create_tsd_exported_runtime_methods(metadata): return utils.read_file(in_temp(f'{file}.d.ts')) -def create_tsd(metadata, embind_tsd): +def create_tsd(metadata, embind_tsd, bindgen_tsd): out = '// TypeScript bindings for emscripten-generated code. Automatically generated at compile time.\n' if settings.EXPORTED_RUNTIME_METHODS: out += create_tsd_exported_runtime_methods(metadata) @@ -712,6 +712,10 @@ def create_tsd(metadata, embind_tsd): # Add in embind definitions. if embind_tsd: export_interfaces += ' & EmbindModule' + if settings.WASM_BINDGEN and bindgen_tsd: + for file_path in bindgen_tsd: + out += utils.read_file(file_path) + export_interfaces += ' & BindgenModule' out += f'export type MainModule = {export_interfaces};\n' if settings.MODULARIZE: return_type = 'MainModule' diff --git a/tools/link.py b/tools/link.py index 3d3d838da3733..1eb1595005f47 100644 --- a/tools/link.py +++ b/tools/link.py @@ -4,6 +4,7 @@ # found in the LICENSE file. import base64 +import glob import json import logging import os @@ -41,10 +42,11 @@ settings, user_settings, ) -from .shared import DEBUG, DYLIB_EXTENSIONS, do_replace, in_temp +from .shared import DEBUG, DYLIB_EXTENSIONS, do_replace, get_emscripten_temp_dir, in_temp from .toolchain_profiler import ToolchainProfiler from .utils import ( WINDOWS, + LinkFlag, delete_file, exit_with_error, get_file_suffix, @@ -1834,7 +1836,7 @@ def phase_calculate_system_libraries(options): @ToolchainProfiler.profile_block('link') -def phase_link(linker_args, wasm_target, js_syms): +def phase_link(linker_args, linker_inputs, wasm_target, js_syms): logger.debug(f'linking: {linker_args}') # Make a final pass over settings.EXPORTED_FUNCTIONS to remove any @@ -1856,11 +1858,12 @@ def phase_link(linker_args, wasm_target, js_syms): # TODO(sbc): Remove this double execution of wasm-ld if we ever find a way to # distinguish EMSCRIPTEN_KEEPALIVE exports from `--export-dynamic` exports. settings.LINKABLE = False - building.link_lld(linker_args, wasm_target, external_symbols=js_syms) + building.link_lld(linker_args, wasm_target, external_symbols=js_syms, + linker_inputs=linker_inputs) settings.LINKABLE = True rtn = extract_metadata.extract_metadata(wasm_target) - building.link_lld(linker_args, wasm_target, external_symbols=js_syms) + building.link_lld(linker_args, wasm_target, external_symbols=js_syms, linker_inputs=linker_inputs) return rtn @@ -1882,6 +1885,10 @@ def phase_post_link(options, in_wasm, wasm_target, target, js_syms, base_metadat settings.TARGET_JS_NAME = os.path.basename(js_target) + if settings.WASM_BINDGEN: + bindgen_jslib = building.run_wasm_bindgen(in_wasm) + settings.JS_LIBRARIES.append(bindgen_jslib) + metadata = phase_emscript(in_wasm, wasm_target, js_syms, base_metadata) if settings.EMBIND_AOT: @@ -2013,7 +2020,10 @@ def phase_emit_tsd(options, wasm_target, js_target, js_syms, metadata): embind_tsd = '' if settings.EMBIND: embind_tsd = run_embind_gen(options, wasm_target, js_syms, {'EMBIND_AOT': False}) - all_tsd = emscripten.create_tsd(metadata, embind_tsd) + bindgen_ts_files = glob.glob(get_emscripten_temp_dir() + "/bindgen_out/*.d.ts", recursive=False) + # This list comprehension then filters out any files that end with .wasm.d.ts. + bindgen_ts_files = [file for file in bindgen_ts_files if not file.endswith('.wasm.d.ts')] + all_tsd = emscripten.create_tsd(metadata, embind_tsd, bindgen_ts_files) out_file = os.path.join(os.path.dirname(js_target), filename) write_file(out_file, all_tsd) @@ -3055,11 +3065,14 @@ def run(options, linker_args): settings.limit_settings(None) if settings.RUNTIME_LINKED_LIBS: - linker_args += settings.RUNTIME_LINKED_LIBS + linker_args += [LinkFlag(f, False) for f in settings.RUNTIME_LINKED_LIBS] if not linker_args: exit_with_error('no input files') + linker_inputs = [f.value for f in linker_args if f.is_file] + linker_args = [f.value for f in linker_args] + if options.output_file and options.output_file.startswith('-'): exit_with_error(f'invalid output filename: `{options.output_file}`') @@ -3117,7 +3130,7 @@ def add_js_deps(sym): settings.ASYNCIFY_IMPORTS_EXCEPT_JS_LIBS = settings.ASYNCIFY_IMPORTS[:] settings.ASYNCIFY_IMPORTS += ['*.' + x for x in js_info['asyncFuncs']] - base_metadata = phase_link(linker_args, wasm_target, js_syms) + base_metadata = phase_link(linker_args, linker_inputs, wasm_target, js_syms) # Special handling for when the user passed '-Wl,--version'. In this case the linker # does not create the output file, but just prints its version and exits with 0. diff --git a/tools/utils.py b/tools/utils.py index 6d1f3d29a8262..18e3aafe77052 100644 --- a/tools/utils.py +++ b/tools/utils.py @@ -17,6 +17,7 @@ import stat import subprocess import sys +from dataclasses import dataclass from pathlib import Path from . import diagnostics @@ -245,3 +246,17 @@ def set_version_globals(): EMSCRIPTEN_VERSION = read_file(filename).strip().strip('"') parts = [int(x) for x in EMSCRIPTEN_VERSION.split('-')[0].split('.')] EMSCRIPTEN_VERSION_MAJOR, EMSCRIPTEN_VERSION_MINOR, EMSCRIPTEN_VERSION_TINY = parts + + +@dataclass +class LinkFlag: + """Used to represent a linker flag. + + The flag value is stored along with a bool that distinguishes input + files from non-files. + + A list of these is returned by separate_linker_flags. + """ + + value: str + is_file: int