From 56abfc02c6e70d0cd89ab6308143f23827004510 Mon Sep 17 00:00:00 2001 From: Shalev Goodman Date: Tue, 7 Oct 2025 04:06:12 +0300 Subject: [PATCH 1/6] Changed orca_wasm to latest version of wirm with parallel feature turned on --- Cargo.lock | 113 +++++-- Cargo.toml | 5 +- .../spidermonkey-embedding-splicer/Cargo.toml | 3 +- .../src/bindgen.rs | 27 ++ .../src/splice.rs | 320 ++++++++++++++++-- .../src/stub_wasi.rs | 16 +- package-lock.json | 4 +- src/componentize.js | 49 ++- 8 files changed, 462 insertions(+), 75 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9078cb2a..e65628de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -188,6 +188,31 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + [[package]] name = "displaydoc" version = "0.2.5" @@ -199,6 +224,12 @@ dependencies = [ "syn", ] +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + [[package]] name = "embedded-io" version = "0.4.0" @@ -227,12 +258,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "fallible-iterator" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" - [[package]] name = "fastrand" version = "2.3.0" @@ -371,9 +396,7 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" dependencies = [ - "fallible-iterator", "indexmap", - "stable_deref_trait", ] [[package]] @@ -642,20 +665,6 @@ version = "1.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cde51589ab56b20a6f686b2c68f7a0bd6add753d697abf720d63f8db3ab7b1ad" -[[package]] -name = "orca-wasm" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86ce9b3c6e21f067e2c1c85a4478ec26dda0a51fec5bc932dfa77f016a880861" -dependencies = [ - "gimli", - "log", - "serde_json", - "tempfile", - "wasm-encoder 0.227.1", - "wasmparser 0.227.1", -] - [[package]] name = "percent-encoding" version = "2.3.1" @@ -729,6 +738,26 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "rustix" version = "1.0.2" @@ -824,10 +853,10 @@ dependencies = [ "clap", "heck", "js-component-bindgen", - "orca-wasm", "rand", "wasm-encoder 0.227.1", - "wasmparser 0.227.1", + "wasmparser 0.239.0", + "wirm", "wit-bindgen", "wit-bindgen-core", "wit-component", @@ -996,6 +1025,16 @@ dependencies = [ "wasmparser 0.227.1", ] +[[package]] +name = "wasm-encoder" +version = "0.239.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5be00faa2b4950c76fe618c409d2c3ea5a3c9422013e079482d78544bb2d184c" +dependencies = [ + "leb128fmt", + "wasmparser 0.239.0", +] + [[package]] name = "wasm-metadata" version = "0.227.1" @@ -1033,6 +1072,18 @@ name = "wasmparser" version = "0.227.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f51cad774fb3c9461ab9bccc9c62dfb7388397b5deda31bf40e8108ccd678b2" +dependencies = [ + "bitflags", + "hashbrown", + "indexmap", + "semver", +] + +[[package]] +name = "wasmparser" +version = "0.239.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c9d90bb93e764f6beabf1d02028c70a2156a6583e63ac4218dd07ef733368b0" dependencies = [ "bitflags", "hashbrown", @@ -1187,6 +1238,20 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "wirm" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14732cb9a0eaf9ec52ecd36b9394ade5c16eea5405d160d8829f0199d97d507d" +dependencies = [ + "log", + "rayon", + "serde_json", + "tempfile", + "wasm-encoder 0.239.0", + "wasmparser 0.239.0", +] + [[package]] name = "wit-bindgen" version = "0.41.0" diff --git a/Cargo.toml b/Cargo.toml index 39e8d176..a2482a99 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,10 +14,11 @@ too_many_arguments = 'allow' anyhow = { version = "1.0.95", default-features = false } heck = { version = "0.5", default-features = false } js-component-bindgen = { version = "1.11.0" } -orca-wasm = { version = "0.9.2", default-features = false } +#orca-wasm = { version = "0.9.2", default-features = false } +wirm = { version = "2.1.0", features = ["parallel"] } rand = { version = "0.8", default-features = false } wasm-encoder = { version = "0.227.1", features = [ "component-model", "std" ] } -wasmparser = { version = "0.227.1", features = ["features", +wasmparser = { version = "0.239.0", features = ["features", "component-model", "hash-collections", "serde", diff --git a/crates/spidermonkey-embedding-splicer/Cargo.toml b/crates/spidermonkey-embedding-splicer/Cargo.toml index be8d7e6d..ab21daea 100644 --- a/crates/spidermonkey-embedding-splicer/Cargo.toml +++ b/crates/spidermonkey-embedding-splicer/Cargo.toml @@ -16,7 +16,8 @@ anyhow = { workspace = true } clap = { version = "4.5.31", features = ["suggestions", "color", "derive"] } heck = { workspace = true } js-component-bindgen = { workspace = true, features = [ "transpile-bindgen" ] } -orca-wasm = { workspace = true } +#orca-wasm = { workspace = true } +wirm = { workspace = true } rand = { workspace = true } wasm-encoder = { workspace = true } wasmparser = { workspace = true } diff --git a/crates/spidermonkey-embedding-splicer/src/bindgen.rs b/crates/spidermonkey-embedding-splicer/src/bindgen.rs index a1216616..4020c834 100644 --- a/crates/spidermonkey-embedding-splicer/src/bindgen.rs +++ b/crates/spidermonkey-embedding-splicer/src/bindgen.rs @@ -2,6 +2,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::fmt::Write; use anyhow::Result; +use std::time::Instant; use heck::*; use js_component_bindgen::function_bindgen::{ ErrHandling, FunctionBindgen, ResourceData, ResourceMap, ResourceTable, @@ -141,6 +142,8 @@ pub fn componentize_bindgen( wid: WorldId, features: &Vec, ) -> Result { + let t_total = Instant::now(); + let mut t_stage = Instant::now(); let mut bindgen = JsBindgen { src: Source::default(), esm_bindgen: EsmBindgen::default(), @@ -164,9 +167,19 @@ pub fn componentize_bindgen( .local_names .exclude_globals(Intrinsic::get_global_names()); + t_stage = Instant::now(); bindgen.imports_bindgen(); + eprintln!( + "trace(bindgen:imports): {} ms", + t_stage.elapsed().as_millis() + ); + t_stage = Instant::now(); bindgen.exports_bindgen()?; + eprintln!( + "trace(bindgen:exports): {} ms", + t_stage.elapsed().as_millis() + ); bindgen.esm_bindgen.populate_export_aliases(); // consolidate import specifiers and generate wrappers @@ -374,18 +387,32 @@ pub fn componentize_bindgen( .concat(), ); + t_stage = Instant::now(); let js_intrinsics = render_intrinsics(&mut bindgen.all_intrinsics, false, true); output.push_str(&js_intrinsics); output.push_str(&bindgen.src); + eprintln!( + "trace(bindgen:intrinsics+emit): {} ms", + t_stage.elapsed().as_millis() + ); import_wrappers .iter() .for_each(|(_, src)| output.push_str(&format!("\n\n{src}"))); + t_stage = Instant::now(); bindgen .esm_bindgen .render_export_imports(&mut output, "$source_mod", &mut bindgen.local_names); + eprintln!( + "trace(bindgen:render-exports-imports): {} ms", + t_stage.elapsed().as_millis() + ); + eprintln!( + "trace(bindgen:total): {} ms", + t_total.elapsed().as_millis() + ); Ok(Componentization { js_bindings: output.to_string(), exports: bindgen.exports, diff --git a/crates/spidermonkey-embedding-splicer/src/splice.rs b/crates/spidermonkey-embedding-splicer/src/splice.rs index 38606a0d..9f97bb02 100644 --- a/crates/spidermonkey-embedding-splicer/src/splice.rs +++ b/crates/spidermonkey-embedding-splicer/src/splice.rs @@ -1,13 +1,14 @@ use std::path::PathBuf; +use std::time::Instant; use anyhow::Result; -use orca_wasm::ir::function::{FunctionBuilder, FunctionModifier}; -use orca_wasm::ir::id::{ExportsID, FunctionID, LocalID}; -use orca_wasm::ir::module::Module; -use orca_wasm::ir::types::{BlockType, ElementItems, InstrumentationMode}; -use orca_wasm::module_builder::AddLocal; -use orca_wasm::opcode::{Inject, InjectAt}; -use orca_wasm::{DataType, Opcode}; +use wirm::ir::function::{FunctionBuilder, FunctionModifier}; +use wirm::ir::id::{ExportsID, FunctionID, LocalID}; +use wirm::ir::module::Module; +use wirm::ir::types::{BlockType, ElementItems, InstrumentationMode}; +use wirm::module_builder::AddLocal; +use wirm::opcode::{Inject, InjectAt}; +use wirm::{DataType, Opcode}; use wasm_encoder::{Encode, Section}; use wasmparser::ExternalKind; use wasmparser::MemArg; @@ -38,13 +39,20 @@ pub fn splice_bindings( world_name: Option, debug: bool, ) -> Result { + let t_total = Instant::now(); + let mut t_stage = Instant::now(); let (mut resolve, id) = match (wit_source, wit_path) { (Some(wit_source), _) => { + t_stage = Instant::now(); let mut resolve = Resolve::default(); let path = PathBuf::from("component.wit"); let id = resolve .push_str(&path, &wit_source) .map_err(|e| e.to_string())?; + eprintln!( + "trace(splice:wit-parse): {} ms", + t_stage.elapsed().as_millis() + ); (resolve, id) } (_, Some(wit_path)) => parse_wit(&wit_path).map_err(|e| format!("{e:?}"))?, @@ -53,15 +61,26 @@ pub fn splice_bindings( } }; + t_stage = Instant::now(); let world = resolve .select_world(id, world_name.as_deref()) .map_err(|e| e.to_string())?; + eprintln!( + "trace(splice:wit-select-world): {} ms", + t_stage.elapsed().as_millis() + ); + t_stage = Instant::now(); let mut wasm_bytes = wit_component::dummy_module(&resolve, world, wit_parser::ManglingAndAbi::Standard32); + eprintln!( + "trace(splice:dummy-module): {} ms", + t_stage.elapsed().as_millis() + ); // merge the engine world with the target world, retaining the engine producers + t_stage = Instant::now(); let (engine_world, producers) = if let Ok(( _, Bindgen { @@ -72,6 +91,10 @@ pub fn splice_bindings( }, )) = decode(&engine) { + eprintln!( + "trace(splice:engine-decode): {} ms", + t_stage.elapsed().as_millis() + ); // we disable the engine run and incoming handler as we recreate these exports // when needed, so remove these from the world before initiating the merge let maybe_run = engine_resolve.worlds[engine_world] @@ -105,25 +128,45 @@ pub fn splice_bindings( .shift_remove(&serve) .unwrap(); } + t_stage = Instant::now(); let map = resolve .merge(engine_resolve) .expect("unable to merge with engine world"); let engine_world = map.map_world(engine_world, None).unwrap(); + eprintln!( + "trace(splice:merge-engine-world): {} ms", + t_stage.elapsed().as_millis() + ); (engine_world, producers) } else { unreachable!(); }; + t_stage = Instant::now(); let componentized = bindgen::componentize_bindgen(&resolve, world, &features).map_err(|err| err.to_string())?; + eprintln!( + "trace(splice:bindgen): {} ms", + t_stage.elapsed().as_millis() + ); + t_stage = Instant::now(); resolve .merge_worlds(engine_world, world) .expect("unable to merge with engine world"); + eprintln!( + "trace(splice:merge-worlds): {} ms", + t_stage.elapsed().as_millis() + ); + t_stage = Instant::now(); let encoded = wit_component::metadata::encode(&resolve, world, StringEncoding::UTF8, producers.as_ref()) .map_err(|e| e.to_string())?; + eprintln!( + "trace(splice:metadata-encode): {} ms", + t_stage.elapsed().as_millis() + ); let section = wasm_encoder::CustomSection { name: "component-type".into(), @@ -257,13 +300,22 @@ pub fn splice_bindings( )); } + t_stage = Instant::now(); let mut wasm = splice::splice(engine, imports, exports, features, debug).map_err(|e| format!("{e:?}"))?; + eprintln!( + "trace(splice:wasm-splice): {} ms", + t_stage.elapsed().as_millis() + ); // add the world section to the spliced wasm wasm.push(section.id()); section.encode(&mut wasm); + eprintln!( + "trace(splice:total): {} ms", + t_total.elapsed().as_millis() + ); Ok(SpliceResult { wasm, exports: componentized @@ -343,41 +395,174 @@ pub fn splice( features: Vec, debug: bool, ) -> Result> { + let t_total = Instant::now(); + let mut t_stage = Instant::now(); + + // Pre-scan sections to understand where bytes are concentrated + { + let t_scan = Instant::now(); + let mut section_sizes: Vec<(String, usize)> = Vec::new(); + let mut custom_sections: usize = 0; + let mut functions_declared: u32 = 0; + let mut data_bytes: usize = 0; + let mut imports_cnt: u32 = 0; + let mut exports_cnt: u32 = 0; + + let mut parser = wasmparser::Parser::new(0); + for payload in parser.parse_all(&engine) { + use wasmparser::Payload::*; + match payload { + Ok(TypeSection(s)) => { + let range = s.range(); + section_sizes.push(("type".into(), range.end - range.start)); + } + Ok(ImportSection(s)) => { + let range = s.range(); + imports_cnt = s.count(); + section_sizes.push(("import".into(), range.end - range.start)); + } + Ok(FunctionSection(s)) => { + let range = s.range(); + functions_declared = s.count(); + section_sizes.push(("function".into(), range.end - range.start)); + } + Ok(TableSection(s)) => { + let range = s.range(); + section_sizes.push(("table".into(), range.end - range.start)); + } + Ok(MemorySection(s)) => { + let range = s.range(); + section_sizes.push(("memory".into(), range.end - range.start)); + } + Ok(GlobalSection(s)) => { + let range = s.range(); + section_sizes.push(("global".into(), range.end - range.start)); + } + Ok(ExportSection(s)) => { + let range = s.range(); + exports_cnt = s.count(); + section_sizes.push(("export".into(), range.end - range.start)); + } + Ok(StartSection { range, .. }) => section_sizes.push(("start".into(), range.end - range.start)), + Ok(ElementSection(s)) => { + let range = s.range(); + section_sizes.push(("element".into(), range.end - range.start)); + } + Ok(CodeSectionStart { .. }) => {} + Ok(CodeSectionEntry(_)) => {} + Ok(DataSection(s)) => { + let range = s.range(); + data_bytes += range.end - range.start; + section_sizes.push(("data".into(), range.end - range.start)); + } + Ok(CustomSection(cs)) => { + let name = cs.name().to_string(); + let size = cs.data().len(); + custom_sections += 1; + section_sizes.push((format!("custom:{name}"), size)); + } + Ok(Version { .. }) | Ok(End(_)) => {}, + Ok(_) => {}, + Err(_) => {} + } + } + // Print a concise summary + eprintln!( + "trace(wasm-splice:scan-sections): {} ms (imports {}, exports {}, decl funcs {}, data {} bytes, custom {})", + t_scan.elapsed().as_millis(), + imports_cnt, + exports_cnt, + functions_declared, + data_bytes, + custom_sections + ); + // Top 5 largest sections + section_sizes.sort_by(|a, b| b.1.cmp(&a.1)); + for (i, (name, sz)) in section_sizes.iter().take(5).enumerate() { + eprintln!("trace(wasm-splice:scan-top{}): {} bytes [{}]", i + 1, sz, name); + } + } + let mut module = Module::parse(&engine, false).unwrap(); + eprintln!( + "trace(wasm-splice:parse): {} ms (engine bytes: {})", + t_stage.elapsed().as_millis(), + engine.len() + ); // since StarlingMonkey implements CLI Run and incoming handler, // we override them only if the guest content exports those functions + t_stage = Instant::now(); remove_if_exported_by_js(&mut module, &exports, "wasi:cli/run@0.2.", "#run"); + eprintln!( + "trace(wasm-splice:remove-run-if-exported): {} ms", + t_stage.elapsed().as_millis() + ); // if 'fetch-event' feature is disabled (default being default-enabled), // remove the built-in incoming-handler which is built around it's use. if !features.contains(&Feature::FetchEvent) { + t_stage = Instant::now(); remove_if_exported_by_js( &mut module, &exports, "wasi:http/incoming-handler@0.2.", "#handle", ); + eprintln!( + "trace(wasm-splice:remove-incoming-handler): {} ms", + t_stage.elapsed().as_millis() + ); } // we reencode the WASI world component data, so strip it out from the // custom section + t_stage = Instant::now(); let maybe_component_section_id = module .custom_sections .get_id("component-type:bindings".to_string()); if let Some(component_section_id) = maybe_component_section_id { module.custom_sections.delete(component_section_id); } + eprintln!( + "trace(wasm-splice:strip-component-section): {} ms", + t_stage.elapsed().as_millis() + ); // extract the native instructions from sample functions // then inline the imported functions and main import gating function // (erasing sample functions in the process) + t_stage = Instant::now(); + let import_cnt = imports.len(); synthesize_import_functions(&mut module, &imports, debug)?; + eprintln!( + "trace(wasm-splice:imports): {} ms ({} imports)", + t_stage.elapsed().as_millis(), + import_cnt + ); // create the exported functions as wrappers around the "cabi_call" function + t_stage = Instant::now(); + let export_cnt = exports.len(); synthesize_export_functions(&mut module, &exports)?; + eprintln!( + "trace(wasm-splice:exports): {} ms ({} exports)", + t_stage.elapsed().as_millis(), + export_cnt + ); - Ok(module.encode()) + t_stage = Instant::now(); + let out = module.encode(); + eprintln!( + "trace(wasm-splice:encode): {} ms (out bytes: {})", + t_stage.elapsed().as_millis(), + out.len() + ); + eprintln!( + "trace(wasm-splice:total): {} ms", + t_total.elapsed().as_millis() + ); + Ok(out) } fn remove_if_exported_by_js( @@ -422,10 +607,13 @@ fn synthesize_import_functions( imports: &[(String, String, CoreFn, Option)], debug: bool, ) -> Result<()> { + let t_total = Instant::now(); + let mut t_stage = Instant::now(); let mut coreabi_get_import: Option = None; let mut cabi_realloc: Option = None; let mut coreabi_sample_ids = Vec::new(); + t_stage = Instant::now(); for (id, expt) in module.exports.iter().enumerate() { match expt.name.as_str() { "coreabi_sample_i32" | "coreabi_sample_i64" | "coreabi_sample_f32" @@ -435,6 +623,10 @@ fn synthesize_import_functions( _ => {} }; } + eprintln!( + "trace(wasm-splice:imports:scan-exports): {} ms", + t_stage.elapsed().as_millis() + ); let memory = 0; @@ -501,6 +693,7 @@ fn synthesize_import_functions( // let mut import_fnids: Vec = Vec::new(); { + let t_loop = Instant::now(); // synthesized native import function parameters (in order) let ctx_arg = coreabi_sample_i32.args[0]; // let argc_arg = coreabi_sample_i32.args[1]; // Unused @@ -717,19 +910,29 @@ fn synthesize_import_functions( let fid = func.finish_module(module); import_fnids.push(fid); } + eprintln!( + "trace(wasm-splice:imports:build-fns): {} ms ({} imports)", + t_loop.elapsed().as_millis(), + imports.len() + ); // extend the main table to include indices for generated imported functions + t_stage = Instant::now(); let table = module.tables.get_mut(main_tid); table.initial += imports.len() as u64; table.maximum = Some(table.maximum.unwrap() + imports.len() as u64); // create imported function table let els = module.elements.iter_mut().next().unwrap(); - if let ElementItems::Functions(ref mut funcs) = &mut els.1 { + if let ElementItems::Functions(ref mut funcs) = &mut els.items { for fid in import_fnids { funcs.push(fid); } } + eprintln!( + "trace(wasm-splice:imports:update-table): {} ms", + t_stage.elapsed().as_millis() + ); } // Populate the import creation function of the form: @@ -741,6 +944,7 @@ fn synthesize_import_functions( // } // { + t_stage = Instant::now(); let coreabi_get_import_fid = get_export_fid(module, &coreabi_get_import.unwrap()); let args = &module @@ -757,46 +961,84 @@ fn synthesize_import_functions( .get_fn_modifier(coreabi_get_import_fid) .unwrap(); - // walk until we get to the const representing the table index - let mut table_instr_idx = 0; - for (idx, instr) in builder.body.instructions.iter_mut().enumerate() { - if let Operator::I32Const { value: ref mut v } = instr.op { - // we specifically need the const "around" 3393 - // which is the coreabi_sample_i32 table offset - if *v < 1000 || *v > 5000 { - continue; + // Find the I32Const base index and compute the delta to new base + let mut table_instr_idx = 0usize; + let mut delta: i32 = 0; + { + let ops_ro = builder.body.instructions.get_ops(); + for (idx, op) in ops_ro.iter().enumerate() { + if let Operator::I32Const { value } = op { + if *value < 1000 || *value > 5000 { + continue; + } + delta = import_fn_table_start_idx - *value; + table_instr_idx = idx; + break; } - *v = import_fn_table_start_idx; - table_instr_idx = idx; - break; } } - builder.inject_at( - table_instr_idx, - InstrumentationMode::Before, - Operator::LocalGet { - local_index: *arg_idx, - }, - ); - builder.inject_at( - table_instr_idx + 1, - InstrumentationMode::Before, - Operator::I32Add, + // Inject adjustments after the located instruction: add delta and add arg index + builder + .body + .instructions + .set_current_mode(table_instr_idx, InstrumentationMode::After); + if delta != 0 { + builder + .body + .instructions + .add_instr(table_instr_idx, Operator::I32Const { value: delta }); + builder + .body + .instructions + .add_instr(table_instr_idx, Operator::I32Add); + } + builder + .body + .instructions + .add_instr( + table_instr_idx, + Operator::LocalGet { + local_index: *arg_idx, + }, + ); + builder + .body + .instructions + .add_instr(table_instr_idx, Operator::I32Add); + builder + .body + .instructions + .finish_instr(table_instr_idx); + eprintln!( + "trace(wasm-splice:imports:fixup-get-import): {} ms", + t_stage.elapsed().as_millis() ); } // remove unnecessary exports + t_stage = Instant::now(); module.exports.delete(coreabi_to_bigint64); module.exports.delete(coreabi_from_bigint64); module.exports.delete(coreabi_get_import.unwrap()); for id in coreabi_sample_ids { module.exports.delete(id); } + eprintln!( + "trace(wasm-splice:imports:prune-exports): {} ms", + t_stage.elapsed().as_millis() + ); + + eprintln!( + "trace(wasm-splice:imports:total): {} ms", + t_total.elapsed().as_millis() + ); Ok(()) } fn synthesize_export_functions(module: &mut Module, exports: &[(String, CoreFn)]) -> Result<()> { + let t_total = Instant::now(); + let mut t_stage = Instant::now(); let cabi_realloc = get_export_fid( module, &module @@ -817,6 +1059,7 @@ fn synthesize_export_functions(module: &mut Module, exports: &[(String, CoreFn)] let memory = 0; // (2) Export call function synthesis + let t_loop = Instant::now(); for (export_num, (expt_name, expt_sig)) in exports.iter().enumerate() { // Export function synthesis { @@ -1011,10 +1254,25 @@ fn synthesize_export_functions(module: &mut Module, exports: &[(String, CoreFn)] .exports .add_export_func(format!("cabi_post_{expt_name}"), *fid); } + eprintln!( + "trace(wasm-splice:exports:build-fns): {} ms ({} exports)", + t_loop.elapsed().as_millis(), + exports.len() + ); // remove unnecessary exports + t_stage = Instant::now(); module.exports.delete(call_expt); module.exports.delete(post_call_expt); + eprintln!( + "trace(wasm-splice:exports:prune-exports): {} ms", + t_stage.elapsed().as_millis() + ); + + eprintln!( + "trace(wasm-splice:exports:total): {} ms", + t_total.elapsed().as_millis() + ); Ok(()) } diff --git a/crates/spidermonkey-embedding-splicer/src/stub_wasi.rs b/crates/spidermonkey-embedding-splicer/src/stub_wasi.rs index 9fe125bd..66f9f73f 100644 --- a/crates/spidermonkey-embedding-splicer/src/stub_wasi.rs +++ b/crates/spidermonkey-embedding-splicer/src/stub_wasi.rs @@ -3,13 +3,13 @@ use std::path::PathBuf; use std::time::{SystemTime, UNIX_EPOCH}; use anyhow::{bail, Result}; -use orca_wasm::ir::function::FunctionBuilder; -use orca_wasm::ir::id::{FunctionID, LocalID}; -use orca_wasm::ir::module::module_functions::FuncKind; -use orca_wasm::ir::types::{BlockType, InitExpr, Value}; -use orca_wasm::module_builder::AddLocal; -use orca_wasm::{DataType, Instructions, Module, Opcode}; -use wasmparser::{MemArg, TypeRef}; +use wirm::ir::function::FunctionBuilder; +use wirm::ir::id::{FunctionID, LocalID}; +use wirm::ir::module::module_functions::FuncKind; +use wirm::ir::types::{BlockType, InitExpr, Value, Instructions}; +use wirm::module_builder::AddLocal; +use wirm::{DataType, InitInstr, Module, Opcode}; +use wasmparser::{MemArg, Operator, TypeRef}; use wit_parser::Resolve; use crate::parse_wit; @@ -206,7 +206,7 @@ fn stub_random(module: &mut Module) -> Result<()> { // create a mutable random seed global let seed_val: i64 = 0; let seed_global = module.add_global( - InitExpr::new(vec![Instructions::Value(Value::I64(seed_val))]), + InitExpr::new(vec![InitInstr::Value(Value::I64(seed_val))]), DataType::I64, true, false, diff --git a/package-lock.json b/package-lock.json index 3961e7bc..c83bda21 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@bytecodealliance/componentize-js", - "version": "0.19.0", + "version": "0.19.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@bytecodealliance/componentize-js", - "version": "0.19.0", + "version": "0.19.1", "workspaces": [ "." ], diff --git a/src/componentize.js b/src/componentize.js index 5bd6c0ee..14cc30d6 100644 --- a/src/componentize.js +++ b/src/componentize.js @@ -62,6 +62,7 @@ export async function componentize( _deprecatedWitWorldOrOpts = undefined, _deprecatedOpts = undefined, ) { + const t_start = Date.now(); let useOriginalSourceFile = true; let jsSource; @@ -142,6 +143,7 @@ export async function componentize( } // Splice the bindigns for the given WIT world into the engine WASM + const t_splice_start = Date.now(); let { wasm, jsBindings, exports, imports } = splicer.spliceBindings( await readFile(engine), [...features], @@ -150,6 +152,10 @@ export async function componentize( worldName, false, ); + const t_splice_end = Date.now(); + console.error( + `trace(node:spliceBindings): ${(t_splice_end - t_splice_start)} ms`, + ); const inputWasmPath = join(workDir, 'in.wasm'); const outputWasmPath = join(workDir, 'out.wasm'); @@ -263,6 +269,8 @@ export async function componentize( let postProcess; const wizerBin = opts.wizerBin ?? wizer; + console.error('trace(node:wizer): starting'); + const t_wizer_start = Date.now(); postProcess = spawnSync( wizerBin, [ @@ -283,6 +291,10 @@ export async function componentize( encoding: 'utf-8', }, ); + const t_wizer_end = Date.now(); + console.error( + `trace(node:wizer): ${(t_wizer_end - t_wizer_start)} ms (includes engine init + snapshot)`, + ); // If the wizer process failed, parse the output and display to the user if (postProcess.status !== 0) { @@ -297,14 +309,20 @@ export async function componentize( } // Read the generated WASM back into memory + const t_read_out_start = Date.now(); const bin = await readFile(outputWasmPath); + const t_read_out_end = Date.now(); + console.error( + `trace(node:read-out-wasm): ${(t_read_out_end - t_read_out_start)} ms`); // Check for initialization errors, by actually executing the binary in // a mini sandbox to get back the initialization state + const t_check_start = Date.now(); const { exports: { check_init }, getStderr, } = await initWasm(bin); + const t_check_mid = Date.now(); // If not in debug mode, clean up if (!debugBindings) { @@ -318,8 +336,13 @@ export async function componentize( workDir, getStderr, ); + const t_check_end = Date.now(); + console.error( + `trace(node:check-init): initWasm ${(t_check_mid - t_check_start)} ms, check_init ${(t_check_end - t_check_mid)} ms`, + ); // After wizening, stub out the wasi imports depending on what features are enabled + const t_stub_start = Date.now(); const finalBin = splicer.stubWasi( bin, [...features], @@ -327,24 +350,36 @@ export async function componentize( maybeWindowsPath(witPath), worldName, ); + const t_stub_end = Date.now(); + console.error( + `trace(node:stubWasi): ${(t_stub_end - t_stub_start)} ms`, + ); if (debugBindings) { await writeFile('binary.wasm', finalBin); } + const t_component_start = Date.now(); + const adapterBytes = await readFile(preview2Adapter); + const rt = await componentNew( + finalBin, + Object.entries({ + wasi_snapshot_preview1: adapterBytes, + }), + false, + ); + const t_component_mid = Date.now(); const component = await metadataAdd( - await componentNew( - finalBin, - Object.entries({ - wasi_snapshot_preview1: await readFile(preview2Adapter), - }), - false, - ), + rt, Object.entries({ language: [['JavaScript', '']], 'processed-by': [['ComponentizeJS', version]], }), ); + const t_component_end = Date.now(); + console.error( + `trace(node:componentize): componentNew ${(t_component_mid - t_component_start)} ms, metadataAdd ${(t_component_end - t_component_mid)} ms`, + ); // Convert CABI import conventions to ESM import conventions imports = imports.map(([specifier, impt]) => From 54bdde632c384db05209d4082239bf188582a566 Mon Sep 17 00:00:00 2001 From: EngoDev Date: Tue, 7 Oct 2025 15:22:21 +0300 Subject: [PATCH 2/6] Running splicer as CLI for multithreading support --- Cargo.lock | 9 +- Cargo.toml | 17 +- Makefile | 10 +- .../spidermonkey-embedding-splicer/Cargo.toml | 1 + .../src/bin/splicer.rs | 108 +- .../src/stub_wasi.rs | 10 +- src/componentize.js | 1086 +++++++++-------- 7 files changed, 734 insertions(+), 507 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e65628de..95ec1372 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -854,7 +854,8 @@ dependencies = [ "heck", "js-component-bindgen", "rand", - "wasm-encoder 0.227.1", + "serde_json", + "wasm-encoder 0.239.0", "wasmparser 0.239.0", "wirm", "wit-bindgen", @@ -1028,8 +1029,6 @@ dependencies = [ [[package]] name = "wasm-encoder" version = "0.239.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5be00faa2b4950c76fe618c409d2c3ea5a3c9422013e079482d78544bb2d184c" dependencies = [ "leb128fmt", "wasmparser 0.239.0", @@ -1082,8 +1081,6 @@ dependencies = [ [[package]] name = "wasmparser" version = "0.239.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c9d90bb93e764f6beabf1d02028c70a2156a6583e63ac4218dd07ef733368b0" dependencies = [ "bitflags", "hashbrown", @@ -1241,8 +1238,6 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "wirm" version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14732cb9a0eaf9ec52ecd36b9394ade5c16eea5405d160d8829f0199d97d507d" dependencies = [ "log", "rayon", diff --git a/Cargo.toml b/Cargo.toml index a2482a99..8ae6a413 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,10 +15,21 @@ anyhow = { version = "1.0.95", default-features = false } heck = { version = "0.5", default-features = false } js-component-bindgen = { version = "1.11.0" } #orca-wasm = { version = "0.9.2", default-features = false } -wirm = { version = "2.1.0", features = ["parallel"] } +wirm = { path = "../wirm/", features = ["parallel"] } rand = { version = "0.8", default-features = false } -wasm-encoder = { version = "0.227.1", features = [ "component-model", "std" ] } -wasmparser = { version = "0.239.0", features = ["features", +serde_json = { version = "1.0", default-features = false, features = ["alloc"] } +# wasm-encoder = { version = "0.227.1", features = [ "component-model", "std" ] } +# wasmparser = { version = "0.239.0", features = ["features", +# "component-model", +# "hash-collections", +# "serde", +# "simd" , +# "std", +# "validate", +# ] } + +wasm-encoder = { path = "../wasm-tools/crates/wasm-encoder/", features = [ "component-model", "std" ] } +wasmparser = { path = "../wasm-tools/crates/wasmparser/", features = ["features", "component-model", "hash-collections", "serde", diff --git a/Makefile b/Makefile index 3ff03015..b9c52c57 100644 --- a/Makefile +++ b/Makefile @@ -13,8 +13,8 @@ endif STARLINGMONKEY_DEPS = $(STARLINGMONKEY_SRC)/cmake/* embedding/* $(STARLINGMONKEY_SRC)/runtime/* $(STARLINGMONKEY_SRC)/builtins/* $(STARLINGMONKEY_SRC)/builtins/*/* $(STARLINGMONKEY_SRC)/builtins/*/*/* $(STARLINGMONKEY_SRC)/include/* all: release -debug: lib/starlingmonkey_embedding.debug.wasm lib/spidermonkey-embedding-splicer.js -release: lib/starlingmonkey_embedding.wasm lib/spidermonkey-embedding-splicer.js +debug: lib/starlingmonkey_embedding.debug.wasm lib/spidermonkey-embedding-splicer.js target/debug/splicer +release: lib/starlingmonkey_embedding.wasm lib/spidermonkey-embedding-splicer.js target/release/splicer lib/spidermonkey-embedding-splicer.js: target/wasm32-wasip1/release/splicer_component.wasm crates/spidermonkey-embedding-splicer/wit/spidermonkey-embedding-splicer.wit | obj lib @$(JCO) new target/wasm32-wasip1/release/splicer_component.wasm -o obj/spidermonkey-embedding-splicer.wasm --wasi-reactor @@ -23,6 +23,12 @@ lib/spidermonkey-embedding-splicer.js: target/wasm32-wasip1/release/splicer_comp target/wasm32-wasip1/release/splicer_component.wasm: Cargo.toml crates/spidermonkey-embedding-splicer/Cargo.toml crates/spidermonkey-embedding-splicer/src/*.rs crates/splicer-component/src/*.rs cargo build --lib --release --target wasm32-wasip1 +target/release/splicer: Cargo.toml crates/spidermonkey-embedding-splicer/Cargo.toml crates/spidermonkey-embedding-splicer/src/*.rs crates/spidermonkey-embedding-splicer/src/bin/splicer.rs + cargo build --bin splicer --release + +target/debug/splicer: Cargo.toml crates/spidermonkey-embedding-splicer/Cargo.toml crates/spidermonkey-embedding-splicer/src/*.rs crates/spidermonkey-embedding-splicer/src/bin/splicer.rs + cargo build --bin splicer + lib/starlingmonkey_embedding.wasm: $(STARLINGMONKEY_DEPS) | lib cmake -B build-release -DCMAKE_BUILD_TYPE=Release make -j16 -C build-release starlingmonkey_embedding diff --git a/crates/spidermonkey-embedding-splicer/Cargo.toml b/crates/spidermonkey-embedding-splicer/Cargo.toml index ab21daea..8eb5080a 100644 --- a/crates/spidermonkey-embedding-splicer/Cargo.toml +++ b/crates/spidermonkey-embedding-splicer/Cargo.toml @@ -19,6 +19,7 @@ js-component-bindgen = { workspace = true, features = [ "transpile-bindgen" ] } #orca-wasm = { workspace = true } wirm = { workspace = true } rand = { workspace = true } +serde_json = { workspace = true } wasm-encoder = { workspace = true } wasmparser = { workspace = true } wit-bindgen = { workspace = true } diff --git a/crates/spidermonkey-embedding-splicer/src/bin/splicer.rs b/crates/spidermonkey-embedding-splicer/src/bin/splicer.rs index 7d62a8d8..fb53dc8d 100644 --- a/crates/spidermonkey-embedding-splicer/src/bin/splicer.rs +++ b/crates/spidermonkey-embedding-splicer/src/bin/splicer.rs @@ -5,7 +5,9 @@ use std::str::FromStr; use anyhow::{Context, Result}; use clap::{Parser, Subcommand}; -use spidermonkey_embedding_splicer::wit::exports::local::spidermonkey_embedding_splicer::splicer::Feature; +use spidermonkey_embedding_splicer::wit::exports::local::spidermonkey_embedding_splicer::splicer::{ + CoreFn, CoreTy, Feature, +}; use spidermonkey_embedding_splicer::{splice, stub_wasi}; #[derive(Parser, Debug)] @@ -139,8 +141,112 @@ fn main() -> Result<()> { out_dir.join("initializer.js").display() ) })?; + + // Write exports and imports as JSON (manual serialization) + let exports_json = serialize_exports(&result.exports); + fs::write(out_dir.join("exports.json"), exports_json).with_context(|| { + format!( + "Failed to write exports file: {}", + out_dir.join("exports.json").display() + ) + })?; + + let imports_json = serialize_imports(&result.imports); + fs::write(out_dir.join("imports.json"), imports_json).with_context(|| { + format!( + "Failed to write imports file: {}", + out_dir.join("imports.json").display() + ) + })?; + + println!( + "Successfully generated bindings and saved to {}", + out_dir.display() + ); } } Ok(()) } + +/// Manually serialize exports to JSON +fn serialize_exports(exports: &[(String, CoreFn)]) -> String { + let mut result = String::from("[\n"); + for (i, (name, core_fn)) in exports.iter().enumerate() { + if i > 0 { + result.push_str(",\n"); + } + result.push_str(" [\""); + result.push_str(&name.replace('\\', "\\\\").replace('"', "\\\"")); + result.push_str("\", "); + result.push_str(&serialize_core_fn(core_fn)); + result.push(']'); + } + result.push_str("\n]"); + result +} + +/// Manually serialize imports to JSON +fn serialize_imports(imports: &[(String, String, u32)]) -> String { + let mut result = String::from("[\n"); + for (i, (specifier, name, arg_count)) in imports.iter().enumerate() { + if i > 0 { + result.push_str(",\n"); + } + result.push_str(" [\""); + result.push_str(&specifier.replace('\\', "\\\\").replace('"', "\\\"")); + result.push_str("\", \""); + result.push_str(&name.replace('\\', "\\\\").replace('"', "\\\"")); + result.push_str("\", "); + result.push_str(&arg_count.to_string()); + result.push(']'); + } + result.push_str("\n]"); + result +} + +/// Manually serialize CoreFn to JSON +fn serialize_core_fn(core_fn: &CoreFn) -> String { + let mut result = String::from("{"); + + // params + result.push_str("\"params\": ["); + for (i, param) in core_fn.params.iter().enumerate() { + if i > 0 { + result.push_str(", "); + } + result.push_str(&serialize_core_ty(param)); + } + result.push_str("], "); + + // ret + result.push_str("\"ret\": "); + if let Some(ref ret) = core_fn.ret { + result.push_str(&serialize_core_ty(ret)); + } else { + result.push_str("null"); + } + result.push_str(", "); + + // retptr + result.push_str(&format!("\"retptr\": {}, ", core_fn.retptr)); + + // retsize + result.push_str(&format!("\"retsize\": {}, ", core_fn.retsize)); + + // paramptr + result.push_str(&format!("\"paramptr\": {}", core_fn.paramptr)); + + result.push('}'); + result +} + +/// Manually serialize CoreTy to JSON +fn serialize_core_ty(core_ty: &CoreTy) -> String { + match core_ty { + CoreTy::I32 => "\"i32\"".to_string(), + CoreTy::I64 => "\"i64\"".to_string(), + CoreTy::F32 => "\"f32\"".to_string(), + CoreTy::F64 => "\"f64\"".to_string(), + } +} diff --git a/crates/spidermonkey-embedding-splicer/src/stub_wasi.rs b/crates/spidermonkey-embedding-splicer/src/stub_wasi.rs index 66f9f73f..e1159381 100644 --- a/crates/spidermonkey-embedding-splicer/src/stub_wasi.rs +++ b/crates/spidermonkey-embedding-splicer/src/stub_wasi.rs @@ -3,13 +3,13 @@ use std::path::PathBuf; use std::time::{SystemTime, UNIX_EPOCH}; use anyhow::{bail, Result}; +use wasmparser::{MemArg, TypeRef}; use wirm::ir::function::FunctionBuilder; use wirm::ir::id::{FunctionID, LocalID}; use wirm::ir::module::module_functions::FuncKind; -use wirm::ir::types::{BlockType, InitExpr, Value, Instructions}; +use wirm::ir::types::{BlockType, InitExpr, Value}; use wirm::module_builder::AddLocal; use wirm::{DataType, InitInstr, Module, Opcode}; -use wasmparser::{MemArg, Operator, TypeRef}; use wit_parser::Resolve; use crate::parse_wit; @@ -44,14 +44,14 @@ where }; let ty = module.types.get(ty_id).unwrap(); - let (params, results) = (ty.params().to_vec(), ty.results().to_vec()); - let mut builder = FunctionBuilder::new(params.as_slice(), results.as_slice()); + let mut builder = FunctionBuilder::new(ty.params().as_slice(), ty.results().as_slice()); let _args = stub(&mut builder)?; builder.replace_import_in_module(module, iid); return Ok(Some(fid)); } + Ok(None) } @@ -120,7 +120,9 @@ pub fn stub_wasi( target_world_imports.insert(resolve.name_canonicalized_world_key(key)); } + let start_time = std::time::Instant::now(); let mut module = Module::parse(wasm.as_slice(), false).unwrap(); + println!("stub_wasi module parse took {:?}", start_time.elapsed()); stub_preview1(&mut module)?; diff --git a/src/componentize.js b/src/componentize.js index 14cc30d6..d9a7a64b 100644 --- a/src/componentize.js +++ b/src/componentize.js @@ -13,17 +13,15 @@ import { createHash } from 'node:crypto'; import oxc from 'oxc-parser'; import wizer from '@bytecodealliance/wizer'; import { - componentNew, - metadataAdd, - preview1AdapterReactorPath, + componentNew, + metadataAdd, + preview1AdapterReactorPath, } from '@bytecodealliance/jco'; -import { splicer } from '../lib/spidermonkey-embedding-splicer.js'; - import { maybeWindowsPath } from './platform.js'; export const { version } = JSON.parse( - await readFile(new URL('../package.json', import.meta.url), 'utf8'), + await readFile(new URL('../package.json', import.meta.url), 'utf8'), ); /** Prefix into wizer error output that indicates a error/trap */ @@ -45,363 +43,458 @@ const CHECK_INIT_RETURN_TYPE_PARSE = 2; /** Default settings for debug options */ const DEFAULT_DEBUG_SETTINGS = { - bindings: false, - bindingsDir: null, + bindings: false, + bindingsDir: null, - binary: false, - binaryPath: null, + binary: false, + binaryPath: null, - wizerLogging: false, + wizerLogging: false, }; /** Features that are used by default if not explicitly disabled */ export const DEFAULT_FEATURES = ['stdio', 'random', 'clocks', 'http', 'fetch-event']; export async function componentize( - opts, - _deprecatedWitWorldOrOpts = undefined, - _deprecatedOpts = undefined, + opts, + _deprecatedWitWorldOrOpts = undefined, + _deprecatedOpts = undefined, ) { - const t_start = Date.now(); - let useOriginalSourceFile = true; - let jsSource; - - // Handle the two old signatures - // (jsSource, witWorld, opts?) - // (jsSource, opts) - if (typeof opts === 'string') { - jsSource = opts; - useOriginalSourceFile = false; - if (typeof _deprecatedWitWorldOrOpts === 'string') { - opts = _deprecatedOpts || {}; - opts.witWorld = _deprecatedWitWorldOrOpts; - } else { - if (typeof _deprecatedWitWorldOrOpts !== 'object') { - throw new Error( - `componentize: second argument must be an object or a string, but is ${typeof _deprecatedWitWorldOrOpts}`, - ); - } - opts = _deprecatedWitWorldOrOpts; - } - } - - // Prepare a working directory for use during componentization - const { sourcesDir, baseDir: workDir } = await prepWorkDir(); - - let { - sourceName = 'source.js', - sourcePath = maybeWindowsPath(join(sourcesDir, sourceName)), - preview2Adapter = preview1AdapterReactorPath(), - witPath, - witWorld, - worldName, - disableFeatures = [], - enableFeatures = [], - - debug = { ...DEFAULT_DEBUG_SETTINGS }, - debugBuild = false, - debugBindings = false, - enableWizerLogging = false, - - runtimeArgs, - - } = opts; - - debugBindings = debugBindings || debug?.bindings; - debugBuild = debugBuild || debug?.build; - enableWizerLogging = enableWizerLogging || debug?.enableWizerLogging; - - // Determine the path to the StarlingMonkey binary - const engine = getEnginePath(opts); - - // Determine the default features that should be included - const features = new Set(); - for (let f of DEFAULT_FEATURES) { - if (!disableFeatures.includes(f)) { - features.add(f); - } - } - - if (!jsSource && sourcePath) { - jsSource = await readFile(sourcePath, 'utf8'); - } - const detectedExports = await detectKnownSourceExportNames( - sourceName, - jsSource, - ); - - // If there is an export of incomingHandler, there is likely to be a - // manual implementation of wasi:http/incoming-handler, so we should - // disable fetch-event - if (features.has('http') && detectedExports.has('incomingHandler')) { - if (debugBindings) { - console.error( - 'Detected `incomingHandler` export, disabling fetch-event...', - ); - } - features.delete('fetch-event'); - } - - // Splice the bindigns for the given WIT world into the engine WASM - const t_splice_start = Date.now(); - let { wasm, jsBindings, exports, imports } = splicer.spliceBindings( - await readFile(engine), - [...features], - witWorld, - maybeWindowsPath(witPath), - worldName, - false, - ); - const t_splice_end = Date.now(); - console.error( - `trace(node:spliceBindings): ${(t_splice_end - t_splice_start)} ms`, - ); - - const inputWasmPath = join(workDir, 'in.wasm'); - const outputWasmPath = join(workDir, 'out.wasm'); - - await writeFile(inputWasmPath, Buffer.from(wasm)); - let initializerPath = maybeWindowsPath(join(sourcesDir, 'initializer.js')); - await writeFile(initializerPath, jsBindings); - - if (debugBindings) { - // If a bindings output directory was specified, output generated bindings to files - if (debug?.bindingsDir) { - console.error(`Storing debug files in "${debug?.bindingsDir}"\n`); - // Ensure the debug bindings dir exists, and is a directory - if (!(await stat(debug?.bindingsDir).then((s) => s.isDirectory()))) { - throw new Error( - `Missing/invalid debug bindings directory [${debug?.bindingsDir}]`, - ); - } - // Write debug to bindings debug directory - await Promise.all([ - writeFile(join(debug?.bindingsDir, 'source.debug.js'), jsSource), - writeFile(join(debug?.bindingsDir, 'bindings.debug.js'), jsBindings), - writeFile( - join(debug?.bindingsDir, 'imports.debug.json'), - JSON.stringify(imports, null, 2), - ), - writeFile( - join(debug?.bindingsDir, 'exports.debug.json'), - JSON.stringify(exports, null, 2), - ), - ]); - } else { - // If a bindings output directory was not specified, output to stdout - console.error('--- JS Bindings ---'); - console.error( - jsBindings - .split('\n') - .map((ln, idx) => `${(idx + 1).toString().padStart(4, ' ')} | ${ln}`) - .join('\n'), - ); - console.error('--- JS Imports ---'); - console.error(imports); - console.error('--- JS Exports ---'); - console.error(exports); - } - } - - if (!useOriginalSourceFile) { - if (debugBindings) { - console.error(`> Writing JS source to ${tmpDir}/sources`); - } - await writeFile(sourcePath, jsSource); - } - - let hostenv = {}; - - if (opts.env) { - hostenv = typeof opts.env === 'object' ? opts.env : process.env; - } - - const env = { - ...hostenv, - DEBUG: enableWizerLogging ? '1' : '', - SOURCE_NAME: sourceName, - EXPORT_CNT: exports.length.toString(), - FEATURE_CLOCKS: features.has('clocks') ? '1' : '', - }; - - for (const [idx, [export_name, expt]] of exports.entries()) { - env[`EXPORT${idx}_NAME`] = export_name; - env[`EXPORT${idx}_ARGS`] = - (expt.paramptr ? '*' : '') + expt.params.join(','); - env[`EXPORT${idx}_RET`] = (expt.retptr ? '*' : '') + (expt.ret || ''); - env[`EXPORT${idx}_RETSIZE`] = String(expt.retsize); - } - - for (let i = 0; i < imports.length; i++) { - env[`IMPORT${i}_NAME`] = imports[i][1]; - env[`IMPORT${i}_ARGCNT`] = String(imports[i][2]); - } - env['IMPORT_CNT'] = imports.length; - - if (debugBindings) { - console.error('--- Wizer Env ---'); - console.error(env); - } - - sourcePath = maybeWindowsPath(sourcePath); - let workspacePrefix = dirname(sourcePath); - - // If the source path is within the current working directory, strip the - // cwd as a prefix from the source path, and remap the paths seen by the - // component to be relative to the current working directory. - // This only works in wizer. - if (!useOriginalSourceFile) { - workspacePrefix = sourcesDir; - sourcePath = sourceName; - } - let currentDir = maybeWindowsPath(cwd()); - if (workspacePrefix.startsWith(currentDir)) { - workspacePrefix = currentDir; - sourcePath = sourcePath.slice(workspacePrefix.length + 1); - } - - let args = `--initializer-script-path ${initializerPath} --strip-path-prefix ${workspacePrefix}/ ${sourcePath}`; - runtimeArgs = runtimeArgs ? `${runtimeArgs} ${args}` : args; - - let preopens = [`--dir ${sourcesDir}`]; - preopens.push(`--mapdir /::${workspacePrefix}`); - - let postProcess; - - const wizerBin = opts.wizerBin ?? wizer; - console.error('trace(node:wizer): starting'); - const t_wizer_start = Date.now(); - postProcess = spawnSync( - wizerBin, - [ - '--allow-wasi', - '--init-func', - 'componentize.wizer', - ...preopens, - `--wasm-bulk-memory=true`, - '--inherit-env=true', - `-o=${outputWasmPath}`, - inputWasmPath, - ], - { - stdio: [null, stdout, 'pipe'], - env, - input: runtimeArgs, - shell: true, - encoding: 'utf-8', - }, - ); - const t_wizer_end = Date.now(); - console.error( - `trace(node:wizer): ${(t_wizer_end - t_wizer_start)} ms (includes engine init + snapshot)`, - ); - - // If the wizer process failed, parse the output and display to the user - if (postProcess.status !== 0) { - let wizerErr = parseWizerStderr(postProcess.stderr); - let err = `Failed to initialize component:\n${wizerErr}`; - if (debugBindings) { - err += `\n\nBinary and sources available for debugging at ${workDir}\n`; - } else { - await rm(workDir, { recursive: true }); - } - throw new Error(err); - } - - // Read the generated WASM back into memory - const t_read_out_start = Date.now(); - const bin = await readFile(outputWasmPath); - const t_read_out_end = Date.now(); - console.error( - `trace(node:read-out-wasm): ${(t_read_out_end - t_read_out_start)} ms`); - - // Check for initialization errors, by actually executing the binary in - // a mini sandbox to get back the initialization state - const t_check_start = Date.now(); - const { - exports: { check_init }, - getStderr, - } = await initWasm(bin); - const t_check_mid = Date.now(); - - // If not in debug mode, clean up - if (!debugBindings) { - await rm(workDir, { recursive: true }); - } - - /// Process output of check init, throwing if necessary - await handleCheckInitOutput( - check_init(), - initializerPath, - workDir, - getStderr, - ); - const t_check_end = Date.now(); - console.error( - `trace(node:check-init): initWasm ${(t_check_mid - t_check_start)} ms, check_init ${(t_check_end - t_check_mid)} ms`, - ); - - // After wizening, stub out the wasi imports depending on what features are enabled - const t_stub_start = Date.now(); - const finalBin = splicer.stubWasi( - bin, - [...features], - witWorld, - maybeWindowsPath(witPath), - worldName, - ); - const t_stub_end = Date.now(); - console.error( - `trace(node:stubWasi): ${(t_stub_end - t_stub_start)} ms`, - ); - - if (debugBindings) { - await writeFile('binary.wasm', finalBin); - } - - const t_component_start = Date.now(); - const adapterBytes = await readFile(preview2Adapter); - const rt = await componentNew( - finalBin, - Object.entries({ - wasi_snapshot_preview1: adapterBytes, - }), - false, - ); - const t_component_mid = Date.now(); - const component = await metadataAdd( - rt, - Object.entries({ - language: [['JavaScript', '']], - 'processed-by': [['ComponentizeJS', version]], - }), - ); - const t_component_end = Date.now(); - console.error( - `trace(node:componentize): componentNew ${(t_component_mid - t_component_start)} ms, metadataAdd ${(t_component_end - t_component_mid)} ms`, - ); - - // Convert CABI import conventions to ESM import conventions - imports = imports.map(([specifier, impt]) => - specifier === '$root' ? [impt, 'default'] : [specifier, impt], - ); - - // Build debug object to return - let debugOutput; - if (debugBindings) { - debugOutput.bindings = debug.bindings; - debugOutput.workDir = workDir; - } - if (debug?.binary) { - debugOutput.binary = debug.binary; - debugOutput.binaryPath = debug.binaryPath; - } - - return { - component, - imports, - debug: debugOutput, - }; + const t_start = Date.now(); + let useOriginalSourceFile = true; + let jsSource; + + // Handle the two old signatures + // (jsSource, witWorld, opts?) + // (jsSource, opts) + if (typeof opts === 'string') { + jsSource = opts; + useOriginalSourceFile = false; + if (typeof _deprecatedWitWorldOrOpts === 'string') { + opts = _deprecatedOpts || {}; + opts.witWorld = _deprecatedWitWorldOrOpts; + } else { + if (typeof _deprecatedWitWorldOrOpts !== 'object') { + throw new Error( + `componentize: second argument must be an object or a string, but is ${typeof _deprecatedWitWorldOrOpts}`, + ); + } + opts = _deprecatedWitWorldOrOpts; + } + } + + // Prepare a working directory for use during componentization + const { sourcesDir, baseDir: workDir } = await prepWorkDir(); + + let { + sourceName = 'source.js', + sourcePath = maybeWindowsPath(join(sourcesDir, sourceName)), + preview2Adapter = preview1AdapterReactorPath(), + witPath, + witWorld, + worldName, + disableFeatures = [], + enableFeatures = [], + + debug = { ...DEFAULT_DEBUG_SETTINGS }, + debugBuild = false, + debugBindings = false, + enableWizerLogging = false, + + runtimeArgs, + + } = opts; + + debugBindings = debugBindings || debug?.bindings; + debugBuild = debugBuild || debug?.build; + enableWizerLogging = enableWizerLogging || debug?.enableWizerLogging; + + // Determine the path to the StarlingMonkey binary + const engine = getEnginePath(opts); + + // Determine the default features that should be included + const features = new Set(); + for (let f of DEFAULT_FEATURES) { + if (!disableFeatures.includes(f)) { + features.add(f); + } + } + + if (!jsSource && sourcePath) { + jsSource = await readFile(sourcePath, 'utf8'); + } + const detectedExports = await detectKnownSourceExportNames( + sourceName, + jsSource, + ); + + // If there is an export of incomingHandler, there is likely to be a + // manual implementation of wasi:http/incoming-handler, so we should + // disable fetch-event + if (features.has('http') && detectedExports.has('incomingHandler')) { + if (debugBindings) { + console.error( + 'Detected `incomingHandler` export, disabling fetch-event...', + ); + } + features.delete('fetch-event'); + } + + // Splice the bindings for the given WIT world into the engine WASM + const t_splice_start = Date.now(); + + // Prepare temporary directory for splicer output + const splicerOutDir = join(workDir, 'splicer-out'); + await mkdir(splicerOutDir); + + // Write engine wasm to temp file + const engineInputPath = join(workDir, 'engine.wasm'); + await writeFile(engineInputPath, await readFile(engine)); + + // Get splicer binary path + const splicerBin = getSplicerPath(opts); + + // Build splicer CLI arguments + const splicerArgs = [ + 'splice-bindings', + '--input', engineInputPath, + '--out-dir', splicerOutDir, + ]; + + // Add features + for (const feature of features) { + splicerArgs.push('--features', feature); + } + + // Add WIT path if provided + if (witPath) { + splicerArgs.push('--wit-path', maybeWindowsPath(witPath)); + } + + // Add world name if provided + if (worldName) { + splicerArgs.push('--world-name', worldName); + } + + // Add debug flag if needed + if (debugBindings) { + splicerArgs.push('--debug'); + } + + // Run splicer CLI + console.error('trace(node:splicer): starting'); + const splicerProcess = spawnSync(splicerBin, splicerArgs, { + stdio: ['pipe', 'inherit', 'inherit'], + encoding: 'utf-8', + }); + + if (splicerProcess.status !== 0) { + const err = `Failed to splice bindings: splicer exited with status ${splicerProcess.status}`; + if (debugBindings) { + console.error(`\n\nBinary and sources available for debugging at ${workDir}\n`); + } else { + await rm(workDir, { recursive: true }); + } + throw new Error(err); + } + + // Read the outputs + const wasm = await readFile(join(splicerOutDir, 'component.wasm')); + const jsBindings = await readFile(join(splicerOutDir, 'initializer.js'), 'utf8'); + const exports = JSON.parse(await readFile(join(splicerOutDir, 'exports.json'), 'utf8')); + let imports = JSON.parse(await readFile(join(splicerOutDir, 'imports.json'), 'utf8')); + + const t_splice_end = Date.now(); + console.error( + `trace(node:spliceBindings): ${(t_splice_end - t_splice_start)} ms`, + ); + + const inputWasmPath = join(workDir, 'in.wasm'); + const outputWasmPath = join(workDir, 'out.wasm'); + + await writeFile(inputWasmPath, Buffer.from(wasm)); + let initializerPath = maybeWindowsPath(join(sourcesDir, 'initializer.js')); + await writeFile(initializerPath, jsBindings); + + if (debugBindings) { + // If a bindings output directory was specified, output generated bindings to files + if (debug?.bindingsDir) { + console.error(`Storing debug files in "${debug?.bindingsDir}"\n`); + // Ensure the debug bindings dir exists, and is a directory + if (!(await stat(debug?.bindingsDir).then((s) => s.isDirectory()))) { + throw new Error( + `Missing/invalid debug bindings directory [${debug?.bindingsDir}]`, + ); + } + // Write debug to bindings debug directory + await Promise.all([ + writeFile(join(debug?.bindingsDir, 'source.debug.js'), jsSource), + writeFile(join(debug?.bindingsDir, 'bindings.debug.js'), jsBindings), + writeFile( + join(debug?.bindingsDir, 'imports.debug.json'), + JSON.stringify(imports, null, 2), + ), + writeFile( + join(debug?.bindingsDir, 'exports.debug.json'), + JSON.stringify(exports, null, 2), + ), + ]); + } else { + // If a bindings output directory was not specified, output to stdout + console.error('--- JS Bindings ---'); + console.error( + jsBindings + .split('\n') + .map((ln, idx) => `${(idx + 1).toString().padStart(4, ' ')} | ${ln}`) + .join('\n'), + ); + console.error('--- JS Imports ---'); + console.error(imports); + console.error('--- JS Exports ---'); + console.error(exports); + } + } + + if (!useOriginalSourceFile) { + if (debugBindings) { + console.error(`> Writing JS source to ${tmpDir}/sources`); + } + await writeFile(sourcePath, jsSource); + } + + let hostenv = {}; + + if (opts.env) { + hostenv = typeof opts.env === 'object' ? opts.env : process.env; + } + + const env = { + ...hostenv, + DEBUG: enableWizerLogging ? '1' : '', + SOURCE_NAME: sourceName, + EXPORT_CNT: exports.length.toString(), + FEATURE_CLOCKS: features.has('clocks') ? '1' : '', + }; + + for (const [idx, [export_name, expt]] of exports.entries()) { + env[`EXPORT${idx}_NAME`] = export_name; + env[`EXPORT${idx}_ARGS`] = + (expt.paramptr ? '*' : '') + expt.params.join(','); + env[`EXPORT${idx}_RET`] = (expt.retptr ? '*' : '') + (expt.ret || ''); + env[`EXPORT${idx}_RETSIZE`] = String(expt.retsize); + } + + for (let i = 0; i < imports.length; i++) { + env[`IMPORT${i}_NAME`] = imports[i][1]; + env[`IMPORT${i}_ARGCNT`] = String(imports[i][2]); + } + env['IMPORT_CNT'] = imports.length; + + if (debugBindings) { + console.error('--- Wizer Env ---'); + console.error(env); + } + + sourcePath = maybeWindowsPath(sourcePath); + let workspacePrefix = dirname(sourcePath); + + // If the source path is within the current working directory, strip the + // cwd as a prefix from the source path, and remap the paths seen by the + // component to be relative to the current working directory. + // This only works in wizer. + if (!useOriginalSourceFile) { + workspacePrefix = sourcesDir; + sourcePath = sourceName; + } + let currentDir = maybeWindowsPath(cwd()); + if (workspacePrefix.startsWith(currentDir)) { + workspacePrefix = currentDir; + sourcePath = sourcePath.slice(workspacePrefix.length + 1); + } + + let args = `--initializer-script-path ${initializerPath} --strip-path-prefix ${workspacePrefix}/ ${sourcePath}`; + runtimeArgs = runtimeArgs ? `${runtimeArgs} ${args}` : args; + + let preopens = [`--dir ${sourcesDir}`]; + preopens.push(`--mapdir /::${workspacePrefix}`); + + let postProcess; + + const wizerBin = opts.wizerBin ?? wizer; + console.error('trace(node:wizer): starting'); + const t_wizer_start = Date.now(); + postProcess = spawnSync( + wizerBin, + [ + '--allow-wasi', + '--init-func', + 'componentize.wizer', + ...preopens, + `--wasm-bulk-memory=true`, + '--inherit-env=true', + `-o=${outputWasmPath}`, + inputWasmPath, + ], + { + stdio: [null, stdout, 'pipe'], + env, + input: runtimeArgs, + shell: true, + encoding: 'utf-8', + }, + ); + const t_wizer_end = Date.now(); + console.error( + `trace(node:wizer): ${(t_wizer_end - t_wizer_start)} ms (includes engine init + snapshot)`, + ); + + // If the wizer process failed, parse the output and display to the user + if (postProcess.status !== 0) { + let wizerErr = parseWizerStderr(postProcess.stderr); + let err = `Failed to initialize component:\n${wizerErr}`; + if (debugBindings) { + err += `\n\nBinary and sources available for debugging at ${workDir}\n`; + } else { + await rm(workDir, { recursive: true }); + } + throw new Error(err); + } + + // Read the generated WASM back into memory + const t_read_out_start = Date.now(); + const bin = await readFile(outputWasmPath); + const t_read_out_end = Date.now(); + console.error( + `trace(node:read-out-wasm): ${(t_read_out_end - t_read_out_start)} ms`); + + // Check for initialization errors, by actually executing the binary in + // a mini sandbox to get back the initialization state + const t_check_start = Date.now(); + const { + exports: { check_init }, + getStderr, + } = await initWasm(bin); + const t_check_mid = Date.now(); + + /// Process output of check init, throwing if necessary + await handleCheckInitOutput( + check_init(), + initializerPath, + workDir, + getStderr, + ); + const t_check_end = Date.now(); + console.error( + `trace(node:check-init): initWasm ${(t_check_mid - t_check_start)} ms, check_init ${(t_check_end - t_check_mid)} ms`, + ); + + // After wizening, stub out the wasi imports depending on what features are enabled + const t_stub_start = Date.now(); + + // Write wasm to temp file for stubbing + const stubInputPath = join(workDir, 'stub-input.wasm'); + const stubOutputPath = join(workDir, 'stub-output.wasm'); + await writeFile(stubInputPath, bin); + + // Build stub-wasi CLI arguments + const stubArgs = [ + 'stub-wasi', + '--input', stubInputPath, + '--output', stubOutputPath, + ]; + + // Add features + for (const feature of features) { + stubArgs.push('--features', feature); + } + + // Add WIT path if provided + if (witPath) { + stubArgs.push('--wit-path', maybeWindowsPath(witPath)); + } + + // Add world name if provided + if (worldName) { + stubArgs.push('--world-name', worldName); + } + + // Run stub-wasi CLI + console.error('trace(node:stub-wasi): starting'); + const stubProcess = spawnSync(splicerBin, stubArgs, { + stdio: ['pipe', 'inherit', 'inherit'], + encoding: 'utf-8', + }); + + if (stubProcess.status !== 0) { + const err = `Failed to stub WASI: splicer exited with status ${stubProcess.status}`; + if (debugBindings) { + console.error(`\n\nBinary and sources available for debugging at ${workDir}\n`); + } else { + await rm(workDir, { recursive: true }); + } + throw new Error(err); + } + + // Read the stubbed wasm + const finalBin = await readFile(stubOutputPath); + + const t_stub_end = Date.now(); + console.error( + `trace(node:stubWasi): ${(t_stub_end - t_stub_start)} ms`, + ); + + if (debugBindings) { + await writeFile('binary.wasm', finalBin); + } + + const t_component_start = Date.now(); + const adapterBytes = await readFile(preview2Adapter); + const rt = await componentNew( + finalBin, + Object.entries({ + wasi_snapshot_preview1: adapterBytes, + }), + false, + ); + const t_component_mid = Date.now(); + const component = await metadataAdd( + rt, + Object.entries({ + language: [['JavaScript', '']], + 'processed-by': [['ComponentizeJS', version]], + }), + ); + const t_component_end = Date.now(); + console.error( + `trace(node:componentize): componentNew ${(t_component_mid - t_component_start)} ms, metadataAdd ${(t_component_end - t_component_mid)} ms`, + ); + + // Convert CABI import conventions to ESM import conventions + imports = imports.map(([specifier, impt]) => + specifier === '$root' ? [impt, 'default'] : [specifier, impt], + ); + + // Clean up temporary directory unless in debug mode + if (!debugBindings) { + await rm(workDir, { recursive: true }); + } + + // Build debug object to return + let debugOutput; + if (debugBindings) { + debugOutput.bindings = debug.bindings; + debugOutput.workDir = workDir; + } + if (debug?.binary) { + debugOutput.binary = debug.binary; + debugOutput.binaryPath = debug.binaryPath; + } + + return { + component, + imports, + debug: debugOutput, + }; } /** @@ -411,8 +504,8 @@ export async function componentize( * @returns {number} The minimum stack size that should be used as a default. */ function defaultMinStackSize(freeMemoryBytes) { - freeMemoryBytes = freeMemoryBytes ?? freemem(); - return Math.max(8 * 1024 * 1024, Math.floor(freeMemoryBytes * 0.1)); + freeMemoryBytes = freeMemoryBytes ?? freemem(); + return Math.max(8 * 1024 * 1024, Math.floor(freeMemoryBytes * 0.1)); } /** @@ -420,13 +513,13 @@ function defaultMinStackSize(freeMemoryBytes) { * found as line prefixes. */ function stripLinesPrefixes(input, prefixPatterns) { - return input - .split('\n') - .map((line) => - prefixPatterns.reduce((line, n) => line.replace(n, ''), line), - ) - .join('\n') - .trim(); + return input + .split('\n') + .map((line) => + prefixPatterns.reduce((line, n) => line.replace(n, ''), line), + ) + .join('\n') + .trim(); } /** @@ -436,15 +529,15 @@ function stripLinesPrefixes(input, prefixPatterns) { * @returns {string} String that can be printed to describe error output */ function parseWizerStderr(stderr) { - let output = `${stderr}`; - let causeStart = output.indexOf(WIZER_ERROR_CAUSE_PREFIX); - let exitCodeStart = output.indexOf(WIZER_EXIT_CODE_PREFIX); - if (causeStart === -1 || exitCodeStart === -1) { - return output; - } - - let causeEnd = output.indexOf('\n', exitCodeStart + 1); - return `${output.substring(0, causeStart)}${output.substring(causeEnd)}`.trim(); + let output = `${stderr}`; + let causeStart = output.indexOf(WIZER_ERROR_CAUSE_PREFIX); + let exitCodeStart = output.indexOf(WIZER_EXIT_CODE_PREFIX); + if (causeStart === -1 || exitCodeStart === -1) { + return output; + } + + let causeEnd = output.indexOf('\n', exitCodeStart + 1); + return `${output.substring(0, causeStart)}${output.substring(causeEnd)}`.trim(); } /** @@ -454,43 +547,56 @@ function parseWizerStderr(stderr) { * @returns {boolean} whether the value is numeric */ function isNumeric(n) { - switch (typeof n) { - case 'bigint': - case 'number': - return true; - case 'object': - return n.constructor == BigInt || n.constructor == Number; - default: - return false; - } + switch (typeof n) { + case 'bigint': + case 'number': + return true; + case 'object': + return n.constructor == BigInt || n.constructor == Number; + default: + return false; + } } /** Determine the correct path for the engine */ function getEnginePath(opts) { - if (opts.engine) { - return opts.engine; - } - const debugSuffix = opts?.debugBuild ? '.debug' : ''; - let engineBinaryRelPath = `../lib/starlingmonkey_embedding${debugSuffix}.wasm`; + if (opts.engine) { + return opts.engine; + } + const debugSuffix = opts?.debugBuild ? '.debug' : ''; + let engineBinaryRelPath = `../lib/starlingmonkey_embedding${debugSuffix}.wasm`; + + return fileURLToPath(new URL(engineBinaryRelPath, import.meta.url)); +} - return fileURLToPath(new URL(engineBinaryRelPath, import.meta.url)); +/** Determine the correct path for the splicer binary */ +function getSplicerPath(opts) { + if (opts.splicerBin) { + return opts.splicerBin; + } + // The splicer binary should be in target/release or target/debug + const mode = opts?.debugBuild ? 'debug' : 'release'; + const binaryName = platform === 'win32' ? 'splicer.exe' : 'splicer'; + const splicerBinaryRelPath = `../target/${mode}/${binaryName}`; + + return fileURLToPath(new URL(splicerBinaryRelPath, import.meta.url)); } /** Prepare a work directory for use with componentization */ async function prepWorkDir() { - const baseDir = maybeWindowsPath( - join( - tmpdir(), - createHash('sha256') - .update(Math.random().toString()) - .digest('hex') - .slice(0, 12), - ), - ); - await mkdir(baseDir); - const sourcesDir = maybeWindowsPath(join(baseDir, 'sources')); - await mkdir(sourcesDir); - return { baseDir, sourcesDir }; + const baseDir = maybeWindowsPath( + join( + tmpdir(), + createHash('sha256') + .update(Math.random().toString()) + .digest('hex') + .slice(0, 12), + ), + ); + await mkdir(baseDir); + const sourcesDir = maybeWindowsPath(join(baseDir, 'sources')); + await mkdir(sourcesDir); + return { baseDir, sourcesDir }; } /** @@ -500,47 +606,47 @@ async function prepWorkDir() { * @throws If a binary is invalid */ async function initWasm(bin) { - const eep = (name) => () => { - throw new Error( - `Internal error: unexpected call to "${name}" during Wasm verification`, - ); - }; - - let stderr = ''; - const wasmModule = await WebAssembly.compile(bin); - - const mockImports = { - wasi_snapshot_preview1: { - fd_write: function (fd, iovs, iovs_len, nwritten) { - if (fd !== 2) return 0; - const mem = new DataView(exports.memory.buffer); - let written = 0; - for (let i = 0; i < iovs_len; i++) { - const bufPtr = mem.getUint32(iovs + i * 8, true); - const bufLen = mem.getUint32(iovs + 4 + i * 8, true); - stderr += new TextDecoder().decode( - new Uint8Array(exports.memory.buffer, bufPtr, bufLen), - ); - written += bufLen; - } - mem.setUint32(nwritten, written, true); - return 1; - }, - }, - }; - - for (const { module, name } of WebAssembly.Module.imports(wasmModule)) { - mockImports[module] = mockImports[module] || {}; - if (!mockImports[module][name]) mockImports[module][name] = eep(name); - } - - const { exports } = await WebAssembly.instantiate(wasmModule, mockImports); - return { - exports, - getStderr() { - return stderr; - }, - }; + const eep = (name) => () => { + throw new Error( + `Internal error: unexpected call to "${name}" during Wasm verification`, + ); + }; + + let stderr = ''; + const wasmModule = await WebAssembly.compile(bin); + + const mockImports = { + wasi_snapshot_preview1: { + fd_write: function (fd, iovs, iovs_len, nwritten) { + if (fd !== 2) return 0; + const mem = new DataView(exports.memory.buffer); + let written = 0; + for (let i = 0; i < iovs_len; i++) { + const bufPtr = mem.getUint32(iovs + i * 8, true); + const bufLen = mem.getUint32(iovs + 4 + i * 8, true); + stderr += new TextDecoder().decode( + new Uint8Array(exports.memory.buffer, bufPtr, bufLen), + ); + written += bufLen; + } + mem.setUint32(nwritten, written, true); + return 1; + }, + }, + }; + + for (const { module, name } of WebAssembly.Module.imports(wasmModule)) { + mockImports[module] = mockImports[module] || {}; + if (!mockImports[module][name]) mockImports[module][name] = eep(name); + } + + const { exports } = await WebAssembly.instantiate(wasmModule, mockImports); + return { + exports, + getStderr() { + return stderr; + }, + }; } /** @@ -552,33 +658,33 @@ async function initWasm(bin) { * @param {() => string} getStderr - A function that resolves to the stderr output of check init */ async function handleCheckInitOutput( - status, - initializerPath, - workDir, - getStderr, + status, + initializerPath, + workDir, + getStderr, ) { - let err = null; - switch (status) { - case CHECK_INIT_RETURN_OK: - break; - case CHECK_INIT_RETURN_FN_LIST: - err = `Unable to extract expected exports list`; - break; - case CHECK_INIT_RETURN_TYPE_PARSE: - err = `Unable to parse the core ABI export types`; - break; - default: - err = `Unknown error during initialization: ${status}`; - } - - if (err) { - let msg = err; - const stderr = getStderr(); - if (stderr) { - msg += `\n${stripLinesPrefixes(stderr, [new RegExp(`${initializerPath}[:\\d]* ?`)], workDir)}`; - } - throw new Error(msg); - } + let err = null; + switch (status) { + case CHECK_INIT_RETURN_OK: + break; + case CHECK_INIT_RETURN_FN_LIST: + err = `Unable to extract expected exports list`; + break; + case CHECK_INIT_RETURN_TYPE_PARSE: + err = `Unable to parse the core ABI export types`; + break; + default: + err = `Unknown error during initialization: ${status}`; + } + + if (err) { + let msg = err; + const stderr = getStderr(); + if (stderr) { + msg += `\n${stripLinesPrefixes(stderr, [new RegExp(`${initializerPath}[:\\d]* ?`)], workDir)}`; + } + throw new Error(msg); + } } /** @@ -589,27 +695,27 @@ async function handleCheckInitOutput( * @returns {Promise} A Promise that resolves to a list of string that represent unversioned interfaces */ async function detectKnownSourceExportNames(filename, code) { - if (!filename) { - throw new Error('missing filename'); - } - if (!code) { - throw new Error('missing JS code'); - } - - const names = new Set(); - - const results = await oxc.parseAsync(filename, code); - if (results.errors.length > 0) { - throw new Error( - `failed to parse JS source, encountered [${results.errors.length}] errors`, - ); - } - - for (const staticExport of results.module.staticExports) { - for (const entry of staticExport.entries) { - names.add(entry.exportName.name); - } - } - - return names; + if (!filename) { + throw new Error('missing filename'); + } + if (!code) { + throw new Error('missing JS code'); + } + + const names = new Set(); + + const results = await oxc.parseAsync(filename, code); + if (results.errors.length > 0) { + throw new Error( + `failed to parse JS source, encountered [${results.errors.length}] errors`, + ); + } + + for (const staticExport of results.module.staticExports) { + for (const entry of staticExport.entries) { + names.add(entry.exportName.name); + } + } + + return names; } From 598042bf80f9aca9c103f2a761fbbd7e4f0e78d7 Mon Sep 17 00:00:00 2001 From: EngoDev Date: Tue, 7 Oct 2025 16:25:09 +0300 Subject: [PATCH 3/6] Changed back to the crates --- Cargo.lock | 8 +++++++- Cargo.toml | 27 ++++++++++++++------------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 95ec1372..e4e827e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -855,7 +855,7 @@ dependencies = [ "js-component-bindgen", "rand", "serde_json", - "wasm-encoder 0.239.0", + "wasm-encoder 0.227.1", "wasmparser 0.239.0", "wirm", "wit-bindgen", @@ -1029,6 +1029,8 @@ dependencies = [ [[package]] name = "wasm-encoder" version = "0.239.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5be00faa2b4950c76fe618c409d2c3ea5a3c9422013e079482d78544bb2d184c" dependencies = [ "leb128fmt", "wasmparser 0.239.0", @@ -1081,6 +1083,8 @@ dependencies = [ [[package]] name = "wasmparser" version = "0.239.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c9d90bb93e764f6beabf1d02028c70a2156a6583e63ac4218dd07ef733368b0" dependencies = [ "bitflags", "hashbrown", @@ -1238,6 +1242,8 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "wirm" version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14732cb9a0eaf9ec52ecd36b9394ade5c16eea5405d160d8829f0199d97d507d" dependencies = [ "log", "rayon", diff --git a/Cargo.toml b/Cargo.toml index 8ae6a413..518fa704 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,21 +15,12 @@ anyhow = { version = "1.0.95", default-features = false } heck = { version = "0.5", default-features = false } js-component-bindgen = { version = "1.11.0" } #orca-wasm = { version = "0.9.2", default-features = false } -wirm = { path = "../wirm/", features = ["parallel"] } +# wirm = { path = "../wirm/", features = ["parallel"] } +wirm = { version = "2.1.0", features = ["parallel"] } rand = { version = "0.8", default-features = false } serde_json = { version = "1.0", default-features = false, features = ["alloc"] } -# wasm-encoder = { version = "0.227.1", features = [ "component-model", "std" ] } -# wasmparser = { version = "0.239.0", features = ["features", -# "component-model", -# "hash-collections", -# "serde", -# "simd" , -# "std", -# "validate", -# ] } - -wasm-encoder = { path = "../wasm-tools/crates/wasm-encoder/", features = [ "component-model", "std" ] } -wasmparser = { path = "../wasm-tools/crates/wasmparser/", features = ["features", +wasm-encoder = { version = "0.227.1", features = [ "component-model", "std" ] } +wasmparser = { version = "0.239.0", features = ["features", "component-model", "hash-collections", "serde", @@ -37,6 +28,16 @@ wasmparser = { path = "../wasm-tools/crates/wasmparser/", features = ["features" "std", "validate", ] } + +# wasm-encoder = { path = "../wasm-tools/crates/wasm-encoder/", features = [ "component-model", "std" ] } +# wasmparser = { path = "../wasm-tools/crates/wasmparser/", features = ["features", +# "component-model", +# "hash-collections", +# "serde", +# "simd" , +# "std", +# "validate", +# ] } wit-bindgen = { version = "0.41.0", features = [ "macros", "async", "realloc" ] } wit-bindgen-core = { version = "0.41.0", default-features = false } wit-component = { version = "0.227.1", features = ["dummy-module"] } From c626f38da7d8690139478d688a55caa52d498129 Mon Sep 17 00:00:00 2001 From: EngoDev Date: Tue, 7 Oct 2025 17:51:09 +0300 Subject: [PATCH 4/6] Added new files and cleaned up --- .../src/splice.rs | 48 ++- src/cli.js | 89 ++--- src/componentize.js | 183 ++++------- src/splicer.js | 250 ++++++++++++++ test/splicer.js | 304 ++++++++++++++++++ 5 files changed, 685 insertions(+), 189 deletions(-) create mode 100644 src/splicer.js create mode 100644 test/splicer.js diff --git a/crates/spidermonkey-embedding-splicer/src/splice.rs b/crates/spidermonkey-embedding-splicer/src/splice.rs index 9f97bb02..946b887f 100644 --- a/crates/spidermonkey-embedding-splicer/src/splice.rs +++ b/crates/spidermonkey-embedding-splicer/src/splice.rs @@ -2,6 +2,10 @@ use std::path::PathBuf; use std::time::Instant; use anyhow::Result; +use wasm_encoder::{Encode, Section}; +use wasmparser::ExternalKind; +use wasmparser::MemArg; +use wasmparser::Operator; use wirm::ir::function::{FunctionBuilder, FunctionModifier}; use wirm::ir::id::{ExportsID, FunctionID, LocalID}; use wirm::ir::module::Module; @@ -9,10 +13,6 @@ use wirm::ir::types::{BlockType, ElementItems, InstrumentationMode}; use wirm::module_builder::AddLocal; use wirm::opcode::{Inject, InjectAt}; use wirm::{DataType, Opcode}; -use wasm_encoder::{Encode, Section}; -use wasmparser::ExternalKind; -use wasmparser::MemArg; -use wasmparser::Operator; use wit_component::metadata::{decode, Bindgen}; use wit_component::StringEncoding; use wit_parser::Resolve; @@ -312,10 +312,7 @@ pub fn splice_bindings( wasm.push(section.id()); section.encode(&mut wasm); - eprintln!( - "trace(splice:total): {} ms", - t_total.elapsed().as_millis() - ); + eprintln!("trace(splice:total): {} ms", t_total.elapsed().as_millis()); Ok(SpliceResult { wasm, exports: componentized @@ -443,7 +440,9 @@ pub fn splice( exports_cnt = s.count(); section_sizes.push(("export".into(), range.end - range.start)); } - Ok(StartSection { range, .. }) => section_sizes.push(("start".into(), range.end - range.start)), + Ok(StartSection { range, .. }) => { + section_sizes.push(("start".into(), range.end - range.start)) + } Ok(ElementSection(s)) => { let range = s.range(); section_sizes.push(("element".into(), range.end - range.start)); @@ -461,8 +460,8 @@ pub fn splice( custom_sections += 1; section_sizes.push((format!("custom:{name}"), size)); } - Ok(Version { .. }) | Ok(End(_)) => {}, - Ok(_) => {}, + Ok(Version { .. }) | Ok(End(_)) => {} + Ok(_) => {} Err(_) => {} } } @@ -479,7 +478,12 @@ pub fn splice( // Top 5 largest sections section_sizes.sort_by(|a, b| b.1.cmp(&a.1)); for (i, (name, sz)) in section_sizes.iter().take(5).enumerate() { - eprintln!("trace(wasm-splice:scan-top{}): {} bytes [{}]", i + 1, sz, name); + eprintln!( + "trace(wasm-splice:scan-top{}): {} bytes [{}]", + i + 1, + sz, + name + ); } } @@ -992,23 +996,17 @@ fn synthesize_import_functions( .instructions .add_instr(table_instr_idx, Operator::I32Add); } - builder - .body - .instructions - .add_instr( - table_instr_idx, - Operator::LocalGet { - local_index: *arg_idx, - }, - ); + builder.body.instructions.add_instr( + table_instr_idx, + Operator::LocalGet { + local_index: *arg_idx, + }, + ); builder .body .instructions .add_instr(table_instr_idx, Operator::I32Add); - builder - .body - .instructions - .finish_instr(table_instr_idx); + builder.body.instructions.finish_instr(table_instr_idx); eprintln!( "trace(wasm-splice:imports:fixup-get-import): {} ms", t_stage.elapsed().as_millis() diff --git a/src/cli.js b/src/cli.js index 39a1846f..5c22570d 100755 --- a/src/cli.js +++ b/src/cli.js @@ -6,55 +6,60 @@ import { writeFile } from 'node:fs/promises'; import { resolve } from 'node:path'; export async function componentizeCmd(jsSource, opts) { - const { component } = await componentize({ - sourcePath: jsSource, - witPath: resolve(opts.wit), - worldName: opts.worldName, - runtimeArgs: opts.runtimeArgs, - disableFeatures: opts.disable, - preview2Adapter: opts.preview2Adapter, - debugBindings: opts.debugBindings, - debugBuild: opts.useDebugBuild, - enableWizerLogging: opts.enableWizerLogging, - }); - await writeFile(opts.out, component); + const { component } = await componentize({ + sourcePath: jsSource, + witPath: resolve(opts.wit), + worldName: opts.worldName, + runtimeArgs: opts.runtimeArgs, + disableFeatures: opts.disable, + preview2Adapter: opts.preview2Adapter, + debugBindings: opts.debugBindings, + debugBuild: opts.useDebugBuild, + enableWizerLogging: opts.enableWizerLogging, + splicerBin: opts.splicerBin, + }); + await writeFile(opts.out, component); } program - .version('0.19.1') - .description('Create a component from a JavaScript module') - .usage(' --wit wit-world.wit -o ') - .argument('', 'JS source file to build') - .requiredOption('-w, --wit ', 'WIT path to build with') - .option('-n, --world-name ', 'WIT world to build') - .option('--runtime-args ', 'arguments to pass to the runtime') - .addOption( - new Option('-d, --disable ', 'disable WASI features').choices( - DEFAULT_FEATURES, - ), - ) - .option( - '--preview2-adapter ', - 'provide a custom preview2 adapter path', - ) - .option('--use-debug-build', 'use a debug build of StarlingMonkey') - .option('--debug-bindings', 'enable debug logging for bindings generation') - .option( - '--enable-wizer-logging', - 'enable debug logging for calls in the generated component', - ) - .requiredOption('-o, --out ', 'output component file') - .action(asyncAction(componentizeCmd)); + .version('0.19.1') + .description('Create a component from a JavaScript module') + .usage(' --wit wit-world.wit -o ') + .argument('', 'JS source file to build') + .requiredOption('-w, --wit ', 'WIT path to build with') + .option('-n, --world-name ', 'WIT world to build') + .option('--runtime-args ', 'arguments to pass to the runtime') + .addOption( + new Option('-d, --disable ', 'disable WASI features').choices( + DEFAULT_FEATURES, + ), + ) + .option( + '--preview2-adapter ', + 'provide a custom preview2 adapter path', + ) + .option('--use-debug-build', 'use a debug build of StarlingMonkey') + .option('--debug-bindings', 'enable debug logging for bindings generation') + .option( + '--enable-wizer-logging', + 'enable debug logging for calls in the generated component', + ) + .option( + '--splicer-bin ', + 'use native CLI splicer for better performance', + ) + .requiredOption('-o, --out ', 'output component file') + .action(asyncAction(componentizeCmd)); program.showHelpAfterError(); program.parse(); function asyncAction(cmd) { - return function () { - const args = [...arguments]; - (async () => { - await cmd.apply(null, args); - })(); - }; + return function () { + const args = [...arguments]; + (async () => { + await cmd.apply(null, args); + })(); + }; } diff --git a/src/componentize.js b/src/componentize.js index d9a7a64b..e20f4ae5 100644 --- a/src/componentize.js +++ b/src/componentize.js @@ -2,7 +2,7 @@ import { freemem } from 'node:os'; import { TextDecoder } from 'node:util'; import { Buffer } from 'node:buffer'; import { fileURLToPath, URL } from 'node:url'; -import { cwd, stdout, platform } from 'node:process'; +import { cwd, stdout } from 'node:process'; import { spawnSync } from 'node:child_process'; import { tmpdir } from 'node:os'; import { resolve, join, dirname } from 'node:path'; @@ -19,6 +19,12 @@ import { } from '@bytecodealliance/jco'; import { maybeWindowsPath } from './platform.js'; +import { + spliceBindingsCli, + stubWasiCli, + spliceBindingsWasm, + stubWasiWasm, +} from './splicer.js'; export const { version } = JSON.parse( await readFile(new URL('../package.json', import.meta.url), 'utf8'), @@ -100,6 +106,7 @@ export async function componentize( debugBuild = false, debugBindings = false, enableWizerLogging = false, + splicerBin, runtimeArgs, @@ -141,73 +148,43 @@ export async function componentize( } // Splice the bindings for the given WIT world into the engine WASM - const t_splice_start = Date.now(); - - // Prepare temporary directory for splicer output - const splicerOutDir = join(workDir, 'splicer-out'); - await mkdir(splicerOutDir); - - // Write engine wasm to temp file - const engineInputPath = join(workDir, 'engine.wasm'); - await writeFile(engineInputPath, await readFile(engine)); - - // Get splicer binary path - const splicerBin = getSplicerPath(opts); - - // Build splicer CLI arguments - const splicerArgs = [ - 'splice-bindings', - '--input', engineInputPath, - '--out-dir', splicerOutDir, - ]; - - // Add features - for (const feature of features) { - splicerArgs.push('--features', feature); - } - - // Add WIT path if provided - if (witPath) { - splicerArgs.push('--wit-path', maybeWindowsPath(witPath)); - } - - // Add world name if provided - if (worldName) { - splicerArgs.push('--world-name', worldName); - } - - // Add debug flag if needed - if (debugBindings) { - splicerArgs.push('--debug'); - } - - // Run splicer CLI - console.error('trace(node:splicer): starting'); - const splicerProcess = spawnSync(splicerBin, splicerArgs, { - stdio: ['pipe', 'inherit', 'inherit'], - encoding: 'utf-8', - }); - - if (splicerProcess.status !== 0) { - const err = `Failed to splice bindings: splicer exited with status ${splicerProcess.status}`; + const engineWasm = await readFile(engine); + + let result; + try { + if (splicerBin != undefined) { + result = await spliceBindingsCli( + engineWasm, + [...features], + witPath, + worldName, + debugBindings, + workDir, + opts, + ); + } else { + result = await spliceBindingsWasm( + engineWasm, + [...features], + witWorld, + witPath, + worldName, + debugBindings, + ); + } + } catch (err) { if (debugBindings) { console.error(`\n\nBinary and sources available for debugging at ${workDir}\n`); } else { await rm(workDir, { recursive: true }); } - throw new Error(err); + throw err; } - - // Read the outputs - const wasm = await readFile(join(splicerOutDir, 'component.wasm')); - const jsBindings = await readFile(join(splicerOutDir, 'initializer.js'), 'utf8'); - const exports = JSON.parse(await readFile(join(splicerOutDir, 'exports.json'), 'utf8')); - let imports = JSON.parse(await readFile(join(splicerOutDir, 'imports.json'), 'utf8')); - - const t_splice_end = Date.now(); - console.error( - `trace(node:spliceBindings): ${(t_splice_end - t_splice_start)} ms`, - ); + + let wasm = result.wasm; + let jsBindings = result.jsBindings; + let exports = result.exports; + let imports = result.imports; const inputWasmPath = join(workDir, 'in.wasm'); const outputWasmPath = join(workDir, 'out.wasm'); @@ -389,59 +366,34 @@ export async function componentize( ); // After wizening, stub out the wasi imports depending on what features are enabled - const t_stub_start = Date.now(); - - // Write wasm to temp file for stubbing - const stubInputPath = join(workDir, 'stub-input.wasm'); - const stubOutputPath = join(workDir, 'stub-output.wasm'); - await writeFile(stubInputPath, bin); - - // Build stub-wasi CLI arguments - const stubArgs = [ - 'stub-wasi', - '--input', stubInputPath, - '--output', stubOutputPath, - ]; - - // Add features - for (const feature of features) { - stubArgs.push('--features', feature); - } - - // Add WIT path if provided - if (witPath) { - stubArgs.push('--wit-path', maybeWindowsPath(witPath)); - } - - // Add world name if provided - if (worldName) { - stubArgs.push('--world-name', worldName); - } - - // Run stub-wasi CLI - console.error('trace(node:stub-wasi): starting'); - const stubProcess = spawnSync(splicerBin, stubArgs, { - stdio: ['pipe', 'inherit', 'inherit'], - encoding: 'utf-8', - }); - - if (stubProcess.status !== 0) { - const err = `Failed to stub WASI: splicer exited with status ${stubProcess.status}`; + let finalBin; + try { + if (splicerBin != undefined) { + finalBin = await stubWasiCli( + bin, + [...features], + witPath, + worldName, + workDir, + opts, + ); + } else { + finalBin = await stubWasiWasm( + bin, + [...features], + witWorld, + witPath, + worldName, + ); + } + } catch (err) { if (debugBindings) { console.error(`\n\nBinary and sources available for debugging at ${workDir}\n`); } else { await rm(workDir, { recursive: true }); } - throw new Error(err); + throw err; } - - // Read the stubbed wasm - const finalBin = await readFile(stubOutputPath); - - const t_stub_end = Date.now(); - console.error( - `trace(node:stubWasi): ${(t_stub_end - t_stub_start)} ms`, - ); if (debugBindings) { await writeFile('binary.wasm', finalBin); @@ -569,19 +521,6 @@ function getEnginePath(opts) { return fileURLToPath(new URL(engineBinaryRelPath, import.meta.url)); } -/** Determine the correct path for the splicer binary */ -function getSplicerPath(opts) { - if (opts.splicerBin) { - return opts.splicerBin; - } - // The splicer binary should be in target/release or target/debug - const mode = opts?.debugBuild ? 'debug' : 'release'; - const binaryName = platform === 'win32' ? 'splicer.exe' : 'splicer'; - const splicerBinaryRelPath = `../target/${mode}/${binaryName}`; - - return fileURLToPath(new URL(splicerBinaryRelPath, import.meta.url)); -} - /** Prepare a work directory for use with componentization */ async function prepWorkDir() { const baseDir = maybeWindowsPath( diff --git a/src/splicer.js b/src/splicer.js new file mode 100644 index 00000000..2db5b587 --- /dev/null +++ b/src/splicer.js @@ -0,0 +1,250 @@ +import { spawnSync } from 'node:child_process'; +import { join } from 'node:path'; +import { readFile, writeFile, mkdir } from 'node:fs/promises'; +import { fileURLToPath } from 'node:url'; +import { platform } from 'node:process'; +import fs from 'node:fs'; + +import { splicer as wasmSplicer } from '../lib/spidermonkey-embedding-splicer.js'; +import { maybeWindowsPath } from './platform.js'; + +/** + * Get the path to the native splicer binary + * @param {Object} opts - Options object + * @returns {string} Path to the splicer binary + */ +function getSplicerBinaryPath(opts) { + if (opts.splicerBin) { + return opts.splicerBin; + } + // The splicer binary should be in target/release or target/debug + const mode = opts?.debugBuild ? 'debug' : 'release'; + const binaryName = platform === 'win32' ? 'splicer.exe' : 'splicer'; + const splicerBinaryRelPath = `../target/${mode}/${binaryName}`; + + return fileURLToPath(new URL(splicerBinaryRelPath, import.meta.url)); +} + +/** + * Run the splicer CLI with the given command and options + * @param {string} command - The splicer command (e.g., 'splice-bindings', 'stub-wasi') + * @param {string[]} baseArgs - Command-specific base arguments + * @param {string[]} features - List of features to enable + * @param {string|null} witPath - Path to WIT file or directory + * @param {string|null} worldName - Name of the world to use + * @param {Object} opts - Additional options (for binary path) + * @returns {Promise} Throws if the command fails + */ +async function runSplicerCli( + command, + baseArgs, + features, + witPath, + worldName, + opts = {}, +) { + const splicerBin = getSplicerBinaryPath(opts); + + const args = [command, ...baseArgs]; + + for (const feature of features) { + args.push('--features', feature); + } + + // Add WIT path if provided + if (witPath) { + args.push('--wit-path', maybeWindowsPath(witPath)); + } + + // Add world name if provided + if (worldName) { + args.push('--world-name', worldName); + } + + // Run splicer CLI + console.error(`trace(splicer-cli:${command}): starting`); + if (!fs.existsSync(splicerBin)) { + throw new Error(`Failed to run splicer '${splicerBin} ${command}': splicer binary not found at ${splicerBin}`); + } + + const process = spawnSync(splicerBin, args, { + stdio: ['pipe', 'inherit', 'inherit'], + encoding: 'utf-8', + }); + + if (process.status !== 0) { + throw new Error(`Failed to run '${splicerBin} ${command}': exited with status ${process.status}`); + } +} + +/** + * Splice bindings using the native CLI binary + * @param {Buffer} engineWasm - The engine WASM binary + * @param {string[]} features - List of features to enable + * @param {string|null} witPath - Path to WIT file or directory + * @param {string|null} worldName - Name of the world to use + * @param {boolean} debug - Enable debug mode + * @param {string} workDir - Working directory for temporary files + * @param {Object} opts - Additional options (for binary path) + * @returns {Promise<{wasm: Buffer, jsBindings: string, exports: Array, imports: Array}>} + */ +export async function spliceBindingsCli( + engineWasm, + features, + witPath, + worldName, + debug, + workDir, + opts = {}, +) { + const t_start = Date.now(); + + // Prepare temporary directory for splicer output + const splicerOutDir = join(workDir, 'splicer-out'); + await mkdir(splicerOutDir); + + // Write engine wasm to temp file + const engineInputPath = join(workDir, 'engine.wasm'); + await writeFile(engineInputPath, engineWasm); + + // Build command-specific arguments + const baseArgs = [ + '--input', engineInputPath, + '--out-dir', splicerOutDir, + ]; + + // Add debug flag if needed + if (debug) { + baseArgs.push('--debug'); + } + + // Run splicer CLI + await runSplicerCli('splice-bindings', baseArgs, features, witPath, worldName, opts); + + // Read the outputs + const wasm = await readFile(join(splicerOutDir, 'component.wasm')); + const jsBindings = await readFile(join(splicerOutDir, 'initializer.js'), 'utf8'); + const exports = JSON.parse(await readFile(join(splicerOutDir, 'exports.json'), 'utf8')); + const imports = JSON.parse(await readFile(join(splicerOutDir, 'imports.json'), 'utf8')); + + const t_end = Date.now(); + console.error(`trace(splicer-cli:splice-bindings): ${t_end - t_start} ms`); + + return { wasm, jsBindings, exports, imports }; +} + +/** + * Stub WASI imports using the native CLI binary + * @param {Buffer} wasmBinary - The WASM binary to stub + * @param {string[]} features - List of features to enable + * @param {string|null} witPath - Path to WIT file or directory + * @param {string|null} worldName - Name of the world to use + * @param {string} workDir - Working directory for temporary files + * @param {Object} opts - Additional options (for binary path) + * @returns {Promise} The stubbed WASM binary + */ +export async function stubWasiCli( + wasmBinary, + features, + witPath, + worldName, + workDir, + opts = {}, +) { + const t_start = Date.now(); + + // Write wasm to temp file for stubbing + const stubInputPath = join(workDir, 'stub-input.wasm'); + const stubOutputPath = join(workDir, 'stub-output.wasm'); + await writeFile(stubInputPath, wasmBinary); + + // Build command-specific arguments + const baseArgs = [ + '--input', stubInputPath, + '--output', stubOutputPath, + ]; + + // Run splicer CLI + await runSplicerCli('stub-wasi', baseArgs, features, witPath, worldName, opts); + + // Read the stubbed wasm + const finalBin = await readFile(stubOutputPath); + + const t_end = Date.now(); + console.error(`trace(splicer-cli:stub-wasi): ${t_end - t_start} ms`); + + return finalBin; +} + +/** + * Splice bindings using the WASM module + * @param {Buffer} engineWasm - The engine WASM binary + * @param {string[]} features - List of features to enable + * @param {string|null} witWorld - WIT world source + * @param {string|null} witPath - Path to WIT file or directory + * @param {string|null} worldName - Name of the world to use + * @param {boolean} debug - Enable debug mode + * @returns {Promise<{wasm: Buffer, jsBindings: string, exports: Array, imports: Array}>} + */ +export async function spliceBindingsWasm( + engineWasm, + features, + witWorld, + witPath, + worldName, + debug, +) { + const t_start = Date.now(); + + const result = wasmSplicer.spliceBindings( + engineWasm, + features, + witWorld, + maybeWindowsPath(witPath), + worldName, + debug, + ); + + const t_end = Date.now(); + console.error(`trace(splicer-wasm:splice-bindings): ${t_end - t_start} ms`); + + return { + wasm: Buffer.from(result.wasm), + jsBindings: result.jsBindings, + exports: result.exports, + imports: result.imports, + }; +} + +/** + * Stub WASI imports using the WASM module + * @param {Buffer} wasmBinary - The WASM binary to stub + * @param {string[]} features - List of features to enable + * @param {string|null} witWorld - WIT world source + * @param {string|null} witPath - Path to WIT file or directory + * @param {string|null} worldName - Name of the world to use + * @returns {Promise} The stubbed WASM binary + */ +export async function stubWasiWasm( + wasmBinary, + features, + witWorld, + witPath, + worldName, +) { + const t_start = Date.now(); + + const result = wasmSplicer.stubWasi( + wasmBinary, + features, + witWorld, + maybeWindowsPath(witPath), + worldName, + ); + + const t_end = Date.now(); + console.error(`trace(splicer-wasm:stub-wasi): ${t_end - t_start} ms`); + + return Buffer.from(result); +} + diff --git a/test/splicer.js b/test/splicer.js new file mode 100644 index 00000000..73e6daec --- /dev/null +++ b/test/splicer.js @@ -0,0 +1,304 @@ +import { fileURLToPath } from 'node:url'; +import { readFile, writeFile, mkdir } from 'node:fs/promises'; +import { existsSync } from 'node:fs'; +import { platform } from 'node:process'; + +import { componentize } from '@bytecodealliance/componentize-js'; +import { transpile } from '@bytecodealliance/jco'; + +import { suite, test, expect } from 'vitest'; + +import { + DEBUG_TRACING_ENABLED, + DEBUG_TEST_ENABLED, + maybeLogging, +} from './util.js'; + +const DEBUG_BINDINGS = process.env.DEBUG_BINDINGS === '1'; + +/** + * Get the path to the splicer binary + */ +function getSplicerBinaryPath() { + const mode = DEBUG_TEST_ENABLED ? 'debug' : 'release'; + const binaryName = platform === 'win32' ? 'splicer.exe' : 'splicer'; + return fileURLToPath( + new URL(`../target/${mode}/${binaryName}`, import.meta.url), + ); +} + +/** + * Helper to create a simple test component + */ +async function createTestComponent(splicerBin, testName) { + const source = ` + export function add(a, b) { + return a + b; + } + `; + + let componentOpts = { + sourceName: `${testName}.js`, + disableFeatures: maybeLogging(['random', 'clocks', 'http', 'stdio']), + debugBuild: DEBUG_TEST_ENABLED, + debugBindings: DEBUG_BINDINGS, + splicerBin, + }; + + // CLI splicer needs a WIT file, WASM splicer can use inline WIT + if (splicerBin) { + // Write WIT to a temp file for CLI splicer + const witContent = ` + package test:test; + + world test { + export add: func(a: s32, b: s32) -> s32; + } + `; + + const witDir = fileURLToPath( + new URL(`./output/${testName}-wit`, import.meta.url), + ); + await mkdir(witDir, { recursive: true }); + await writeFile(`${witDir}/world.wit`, witContent); + + componentOpts.witPath = witDir; + } else { + // WASM splicer can use inline WIT + componentOpts.witWorld = ` + package test:test; + + world test { + export add: func(a: s32, b: s32) -> s32; + } + `; + } + + const { component } = await componentize(source, componentOpts); + + return component; +} + +/** + * Helper to verify a component works correctly + */ +async function verifyComponent(component, testName) { + const map = { + 'wasi:cli-base/*': '@bytecodealliance/preview2-shim/cli-base#*', + 'wasi:clocks/*': '@bytecodealliance/preview2-shim/clocks#*', + 'wasi:filesystem/*': '@bytecodealliance/preview2-shim/filesystem#*', + 'wasi:http/*': '@bytecodealliance/preview2-shim/http#*', + 'wasi:io/*': '@bytecodealliance/preview2-shim/io#*', + 'wasi:logging/*': '@bytecodealliance/preview2-shim/logging#*', + 'wasi:poll/*': '@bytecodealliance/preview2-shim/poll#*', + 'wasi:random/*': '@bytecodealliance/preview2-shim/random#*', + 'wasi:sockets/*': '@bytecodealliance/preview2-shim/sockets#*', + }; + + const { files } = await transpile(component, { + name: testName, + map, + wasiShim: true, + validLiftingOptimization: false, + tracing: DEBUG_TRACING_ENABLED, + }); + + await mkdir(new URL(`./output/${testName}/interfaces`, import.meta.url), { + recursive: true, + }); + + await writeFile( + new URL(`./output/${testName}.component.wasm`, import.meta.url), + component, + ); + + for (const file of Object.keys(files)) { + await writeFile( + new URL(`./output/${testName}/${file}`, import.meta.url), + files[file], + ); + } + + const outputPath = fileURLToPath( + new URL(`./output/${testName}/${testName}.js`, import.meta.url), + ); + + const instance = await import(outputPath); + + // Test the exported function + const result = instance.add(5, 3); + expect(result).toBe(8); + + return instance; +} + +suite('Splicer Integration', async () => { + const splicerBinPath = getSplicerBinaryPath(); + const hasSplicerBinary = existsSync(splicerBinPath); + + if (hasSplicerBinary) { + console.log(`Found splicer binary at: ${splicerBinPath}`); + } else { + console.warn(`Splicer binary not found at: ${splicerBinPath}`); + console.warn('CLI splicer tests will be skipped'); + } + + test.concurrent('CLI splicer produces valid component', async () => { + if (!hasSplicerBinary) { + console.log('Skipping CLI splicer test - binary not found'); + return; + } + + const component = await createTestComponent( + splicerBinPath, + 'splicer-cli-test', + ); + + expect(component).toBeInstanceOf(Uint8Array); + expect(component.length).toBeGreaterThan(0); + + await verifyComponent(component, 'splicer-cli-test'); + }); + + test.concurrent('WASM splicer produces valid component', async () => { + const component = await createTestComponent( + undefined, + 'splicer-wasm-test', + ); + + expect(component).toBeInstanceOf(Uint8Array); + expect(component.length).toBeGreaterThan(0); + + await verifyComponent(component, 'splicer-wasm-test'); + }); + + test.concurrent( + 'CLI and WASM splicers produce functionally equivalent components', + async () => { + if (!hasSplicerBinary) { + console.log( + 'Skipping equivalence test - CLI splicer binary not found', + ); + return; + } + + const cliComponent = await createTestComponent( + splicerBinPath, + 'splicer-cli-equiv', + ); + const wasmComponent = await createTestComponent( + undefined, + 'splicer-wasm-equiv', + ); + + // Both should produce valid components + expect(cliComponent).toBeInstanceOf(Uint8Array); + expect(wasmComponent).toBeInstanceOf(Uint8Array); + + // Verify both work correctly + const cliInstance = await verifyComponent( + cliComponent, + 'splicer-cli-equiv', + ); + const wasmInstance = await verifyComponent( + wasmComponent, + 'splicer-wasm-equiv', + ); + + // Both should produce the same result + expect(cliInstance.add(10, 20)).toBe(30); + expect(wasmInstance.add(10, 20)).toBe(30); + expect(cliInstance.add(10, 20)).toBe(wasmInstance.add(10, 20)); + }, + ); + + test.concurrent('custom splicer binary path works', async () => { + if (!hasSplicerBinary) { + console.log('Skipping custom path test - binary not found'); + return; + } + + // Test with explicit path + const component = await createTestComponent( + splicerBinPath, + 'splicer-custom-path', + ); + + expect(component).toBeInstanceOf(Uint8Array); + await verifyComponent(component, 'splicer-custom-path'); + }); + + test.concurrent('invalid splicer binary path throws error', async () => { + const invalidPath = '/nonexistent/path/to/splicer'; + + await expect( + createTestComponent(invalidPath, 'splicer-invalid-path'), + ).rejects.toThrow(/splicer binary not found/); + }); + + test.concurrent( + 'CLI splicer with WIT path produces valid component', + async () => { + if (!hasSplicerBinary) { + console.log('Skipping WIT path test - binary not found'); + return; + } + + const source = ` + export function add(a, b) { + return a + b; + } + + export function getResult() { + return 42; + } + `; + + const witPath = fileURLToPath(new URL('./wit', import.meta.url)); + + const { component } = await componentize(source, { + sourceName: 'cli-wit-path-test.js', + witPath, + worldName: 'test2', + disableFeatures: maybeLogging([]), + debugBuild: DEBUG_TEST_ENABLED, + debugBindings: DEBUG_BINDINGS, + splicerBin: splicerBinPath, + }); + + expect(component).toBeInstanceOf(Uint8Array); + expect(component.length).toBeGreaterThan(0); + }, + ); + + test.concurrent( + 'WASM splicer with WIT path produces valid component', + async () => { + const source = ` + export function add(a, b) { + return a + b; + } + + export function getResult() { + return 42; + } + `; + + const witPath = fileURLToPath(new URL('./wit', import.meta.url)); + + const { component } = await componentize(source, { + sourceName: 'wasm-wit-path-test.js', + witPath, + worldName: 'test2', + disableFeatures: maybeLogging([]), + debugBuild: DEBUG_TEST_ENABLED, + debugBindings: DEBUG_BINDINGS, + splicerBin: undefined, + }); + + expect(component).toBeInstanceOf(Uint8Array); + expect(component.length).toBeGreaterThan(0); + }, + ); +}); + From ec7e0615853563bb42ab182e164e39220ef81d56 Mon Sep 17 00:00:00 2001 From: EngoDev Date: Tue, 7 Oct 2025 19:02:30 +0300 Subject: [PATCH 5/6] Cleaned up the code and removed tracing --- Cargo.toml | 11 - .../spidermonkey-embedding-splicer/Cargo.toml | 1 - .../src/bindgen.rs | 27 -- .../src/splice.rs | 244 +----------------- .../src/stub_wasi.rs | 2 - src/componentize.js | 22 -- src/splicer.js | 20 -- 7 files changed, 2 insertions(+), 325 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 518fa704..95c764e6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,8 +14,6 @@ too_many_arguments = 'allow' anyhow = { version = "1.0.95", default-features = false } heck = { version = "0.5", default-features = false } js-component-bindgen = { version = "1.11.0" } -#orca-wasm = { version = "0.9.2", default-features = false } -# wirm = { path = "../wirm/", features = ["parallel"] } wirm = { version = "2.1.0", features = ["parallel"] } rand = { version = "0.8", default-features = false } serde_json = { version = "1.0", default-features = false, features = ["alloc"] } @@ -29,15 +27,6 @@ wasmparser = { version = "0.239.0", features = ["features", "validate", ] } -# wasm-encoder = { path = "../wasm-tools/crates/wasm-encoder/", features = [ "component-model", "std" ] } -# wasmparser = { path = "../wasm-tools/crates/wasmparser/", features = ["features", -# "component-model", -# "hash-collections", -# "serde", -# "simd" , -# "std", -# "validate", -# ] } wit-bindgen = { version = "0.41.0", features = [ "macros", "async", "realloc" ] } wit-bindgen-core = { version = "0.41.0", default-features = false } wit-component = { version = "0.227.1", features = ["dummy-module"] } diff --git a/crates/spidermonkey-embedding-splicer/Cargo.toml b/crates/spidermonkey-embedding-splicer/Cargo.toml index 8eb5080a..438408df 100644 --- a/crates/spidermonkey-embedding-splicer/Cargo.toml +++ b/crates/spidermonkey-embedding-splicer/Cargo.toml @@ -16,7 +16,6 @@ anyhow = { workspace = true } clap = { version = "4.5.31", features = ["suggestions", "color", "derive"] } heck = { workspace = true } js-component-bindgen = { workspace = true, features = [ "transpile-bindgen" ] } -#orca-wasm = { workspace = true } wirm = { workspace = true } rand = { workspace = true } serde_json = { workspace = true } diff --git a/crates/spidermonkey-embedding-splicer/src/bindgen.rs b/crates/spidermonkey-embedding-splicer/src/bindgen.rs index 4020c834..a1216616 100644 --- a/crates/spidermonkey-embedding-splicer/src/bindgen.rs +++ b/crates/spidermonkey-embedding-splicer/src/bindgen.rs @@ -2,7 +2,6 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::fmt::Write; use anyhow::Result; -use std::time::Instant; use heck::*; use js_component_bindgen::function_bindgen::{ ErrHandling, FunctionBindgen, ResourceData, ResourceMap, ResourceTable, @@ -142,8 +141,6 @@ pub fn componentize_bindgen( wid: WorldId, features: &Vec, ) -> Result { - let t_total = Instant::now(); - let mut t_stage = Instant::now(); let mut bindgen = JsBindgen { src: Source::default(), esm_bindgen: EsmBindgen::default(), @@ -167,19 +164,9 @@ pub fn componentize_bindgen( .local_names .exclude_globals(Intrinsic::get_global_names()); - t_stage = Instant::now(); bindgen.imports_bindgen(); - eprintln!( - "trace(bindgen:imports): {} ms", - t_stage.elapsed().as_millis() - ); - t_stage = Instant::now(); bindgen.exports_bindgen()?; - eprintln!( - "trace(bindgen:exports): {} ms", - t_stage.elapsed().as_millis() - ); bindgen.esm_bindgen.populate_export_aliases(); // consolidate import specifiers and generate wrappers @@ -387,32 +374,18 @@ pub fn componentize_bindgen( .concat(), ); - t_stage = Instant::now(); let js_intrinsics = render_intrinsics(&mut bindgen.all_intrinsics, false, true); output.push_str(&js_intrinsics); output.push_str(&bindgen.src); - eprintln!( - "trace(bindgen:intrinsics+emit): {} ms", - t_stage.elapsed().as_millis() - ); import_wrappers .iter() .for_each(|(_, src)| output.push_str(&format!("\n\n{src}"))); - t_stage = Instant::now(); bindgen .esm_bindgen .render_export_imports(&mut output, "$source_mod", &mut bindgen.local_names); - eprintln!( - "trace(bindgen:render-exports-imports): {} ms", - t_stage.elapsed().as_millis() - ); - eprintln!( - "trace(bindgen:total): {} ms", - t_total.elapsed().as_millis() - ); Ok(Componentization { js_bindings: output.to_string(), exports: bindgen.exports, diff --git a/crates/spidermonkey-embedding-splicer/src/splice.rs b/crates/spidermonkey-embedding-splicer/src/splice.rs index 946b887f..569ac805 100644 --- a/crates/spidermonkey-embedding-splicer/src/splice.rs +++ b/crates/spidermonkey-embedding-splicer/src/splice.rs @@ -1,5 +1,4 @@ use std::path::PathBuf; -use std::time::Instant; use anyhow::Result; use wasm_encoder::{Encode, Section}; @@ -11,7 +10,7 @@ use wirm::ir::id::{ExportsID, FunctionID, LocalID}; use wirm::ir::module::Module; use wirm::ir::types::{BlockType, ElementItems, InstrumentationMode}; use wirm::module_builder::AddLocal; -use wirm::opcode::{Inject, InjectAt}; +use wirm::opcode::Inject; use wirm::{DataType, Opcode}; use wit_component::metadata::{decode, Bindgen}; use wit_component::StringEncoding; @@ -39,20 +38,13 @@ pub fn splice_bindings( world_name: Option, debug: bool, ) -> Result { - let t_total = Instant::now(); - let mut t_stage = Instant::now(); let (mut resolve, id) = match (wit_source, wit_path) { (Some(wit_source), _) => { - t_stage = Instant::now(); let mut resolve = Resolve::default(); let path = PathBuf::from("component.wit"); let id = resolve .push_str(&path, &wit_source) .map_err(|e| e.to_string())?; - eprintln!( - "trace(splice:wit-parse): {} ms", - t_stage.elapsed().as_millis() - ); (resolve, id) } (_, Some(wit_path)) => parse_wit(&wit_path).map_err(|e| format!("{e:?}"))?, @@ -61,26 +53,15 @@ pub fn splice_bindings( } }; - t_stage = Instant::now(); let world = resolve .select_world(id, world_name.as_deref()) .map_err(|e| e.to_string())?; - eprintln!( - "trace(splice:wit-select-world): {} ms", - t_stage.elapsed().as_millis() - ); - t_stage = Instant::now(); let mut wasm_bytes = wit_component::dummy_module(&resolve, world, wit_parser::ManglingAndAbi::Standard32); - eprintln!( - "trace(splice:dummy-module): {} ms", - t_stage.elapsed().as_millis() - ); // merge the engine world with the target world, retaining the engine producers - t_stage = Instant::now(); let (engine_world, producers) = if let Ok(( _, Bindgen { @@ -91,10 +72,6 @@ pub fn splice_bindings( }, )) = decode(&engine) { - eprintln!( - "trace(splice:engine-decode): {} ms", - t_stage.elapsed().as_millis() - ); // we disable the engine run and incoming handler as we recreate these exports // when needed, so remove these from the world before initiating the merge let maybe_run = engine_resolve.worlds[engine_world] @@ -128,45 +105,25 @@ pub fn splice_bindings( .shift_remove(&serve) .unwrap(); } - t_stage = Instant::now(); let map = resolve .merge(engine_resolve) .expect("unable to merge with engine world"); let engine_world = map.map_world(engine_world, None).unwrap(); - eprintln!( - "trace(splice:merge-engine-world): {} ms", - t_stage.elapsed().as_millis() - ); (engine_world, producers) } else { unreachable!(); }; - t_stage = Instant::now(); let componentized = bindgen::componentize_bindgen(&resolve, world, &features).map_err(|err| err.to_string())?; - eprintln!( - "trace(splice:bindgen): {} ms", - t_stage.elapsed().as_millis() - ); - t_stage = Instant::now(); resolve .merge_worlds(engine_world, world) .expect("unable to merge with engine world"); - eprintln!( - "trace(splice:merge-worlds): {} ms", - t_stage.elapsed().as_millis() - ); - t_stage = Instant::now(); let encoded = wit_component::metadata::encode(&resolve, world, StringEncoding::UTF8, producers.as_ref()) .map_err(|e| e.to_string())?; - eprintln!( - "trace(splice:metadata-encode): {} ms", - t_stage.elapsed().as_millis() - ); let section = wasm_encoder::CustomSection { name: "component-type".into(), @@ -300,19 +257,13 @@ pub fn splice_bindings( )); } - t_stage = Instant::now(); let mut wasm = splice::splice(engine, imports, exports, features, debug).map_err(|e| format!("{e:?}"))?; - eprintln!( - "trace(splice:wasm-splice): {} ms", - t_stage.elapsed().as_millis() - ); // add the world section to the spliced wasm wasm.push(section.id()); section.encode(&mut wasm); - eprintln!("trace(splice:total): {} ms", t_total.elapsed().as_millis()); Ok(SpliceResult { wasm, exports: componentized @@ -392,181 +343,41 @@ pub fn splice( features: Vec, debug: bool, ) -> Result> { - let t_total = Instant::now(); - let mut t_stage = Instant::now(); - - // Pre-scan sections to understand where bytes are concentrated - { - let t_scan = Instant::now(); - let mut section_sizes: Vec<(String, usize)> = Vec::new(); - let mut custom_sections: usize = 0; - let mut functions_declared: u32 = 0; - let mut data_bytes: usize = 0; - let mut imports_cnt: u32 = 0; - let mut exports_cnt: u32 = 0; - - let mut parser = wasmparser::Parser::new(0); - for payload in parser.parse_all(&engine) { - use wasmparser::Payload::*; - match payload { - Ok(TypeSection(s)) => { - let range = s.range(); - section_sizes.push(("type".into(), range.end - range.start)); - } - Ok(ImportSection(s)) => { - let range = s.range(); - imports_cnt = s.count(); - section_sizes.push(("import".into(), range.end - range.start)); - } - Ok(FunctionSection(s)) => { - let range = s.range(); - functions_declared = s.count(); - section_sizes.push(("function".into(), range.end - range.start)); - } - Ok(TableSection(s)) => { - let range = s.range(); - section_sizes.push(("table".into(), range.end - range.start)); - } - Ok(MemorySection(s)) => { - let range = s.range(); - section_sizes.push(("memory".into(), range.end - range.start)); - } - Ok(GlobalSection(s)) => { - let range = s.range(); - section_sizes.push(("global".into(), range.end - range.start)); - } - Ok(ExportSection(s)) => { - let range = s.range(); - exports_cnt = s.count(); - section_sizes.push(("export".into(), range.end - range.start)); - } - Ok(StartSection { range, .. }) => { - section_sizes.push(("start".into(), range.end - range.start)) - } - Ok(ElementSection(s)) => { - let range = s.range(); - section_sizes.push(("element".into(), range.end - range.start)); - } - Ok(CodeSectionStart { .. }) => {} - Ok(CodeSectionEntry(_)) => {} - Ok(DataSection(s)) => { - let range = s.range(); - data_bytes += range.end - range.start; - section_sizes.push(("data".into(), range.end - range.start)); - } - Ok(CustomSection(cs)) => { - let name = cs.name().to_string(); - let size = cs.data().len(); - custom_sections += 1; - section_sizes.push((format!("custom:{name}"), size)); - } - Ok(Version { .. }) | Ok(End(_)) => {} - Ok(_) => {} - Err(_) => {} - } - } - // Print a concise summary - eprintln!( - "trace(wasm-splice:scan-sections): {} ms (imports {}, exports {}, decl funcs {}, data {} bytes, custom {})", - t_scan.elapsed().as_millis(), - imports_cnt, - exports_cnt, - functions_declared, - data_bytes, - custom_sections - ); - // Top 5 largest sections - section_sizes.sort_by(|a, b| b.1.cmp(&a.1)); - for (i, (name, sz)) in section_sizes.iter().take(5).enumerate() { - eprintln!( - "trace(wasm-splice:scan-top{}): {} bytes [{}]", - i + 1, - sz, - name - ); - } - } - let mut module = Module::parse(&engine, false).unwrap(); - eprintln!( - "trace(wasm-splice:parse): {} ms (engine bytes: {})", - t_stage.elapsed().as_millis(), - engine.len() - ); // since StarlingMonkey implements CLI Run and incoming handler, // we override them only if the guest content exports those functions - t_stage = Instant::now(); remove_if_exported_by_js(&mut module, &exports, "wasi:cli/run@0.2.", "#run"); - eprintln!( - "trace(wasm-splice:remove-run-if-exported): {} ms", - t_stage.elapsed().as_millis() - ); // if 'fetch-event' feature is disabled (default being default-enabled), // remove the built-in incoming-handler which is built around it's use. if !features.contains(&Feature::FetchEvent) { - t_stage = Instant::now(); remove_if_exported_by_js( &mut module, &exports, "wasi:http/incoming-handler@0.2.", "#handle", ); - eprintln!( - "trace(wasm-splice:remove-incoming-handler): {} ms", - t_stage.elapsed().as_millis() - ); } // we reencode the WASI world component data, so strip it out from the // custom section - t_stage = Instant::now(); let maybe_component_section_id = module .custom_sections .get_id("component-type:bindings".to_string()); if let Some(component_section_id) = maybe_component_section_id { module.custom_sections.delete(component_section_id); } - eprintln!( - "trace(wasm-splice:strip-component-section): {} ms", - t_stage.elapsed().as_millis() - ); // extract the native instructions from sample functions // then inline the imported functions and main import gating function // (erasing sample functions in the process) - t_stage = Instant::now(); - let import_cnt = imports.len(); synthesize_import_functions(&mut module, &imports, debug)?; - eprintln!( - "trace(wasm-splice:imports): {} ms ({} imports)", - t_stage.elapsed().as_millis(), - import_cnt - ); // create the exported functions as wrappers around the "cabi_call" function - t_stage = Instant::now(); - let export_cnt = exports.len(); synthesize_export_functions(&mut module, &exports)?; - eprintln!( - "trace(wasm-splice:exports): {} ms ({} exports)", - t_stage.elapsed().as_millis(), - export_cnt - ); - t_stage = Instant::now(); - let out = module.encode(); - eprintln!( - "trace(wasm-splice:encode): {} ms (out bytes: {})", - t_stage.elapsed().as_millis(), - out.len() - ); - eprintln!( - "trace(wasm-splice:total): {} ms", - t_total.elapsed().as_millis() - ); - Ok(out) + Ok(module.encode()) } fn remove_if_exported_by_js( @@ -611,13 +422,10 @@ fn synthesize_import_functions( imports: &[(String, String, CoreFn, Option)], debug: bool, ) -> Result<()> { - let t_total = Instant::now(); - let mut t_stage = Instant::now(); let mut coreabi_get_import: Option = None; let mut cabi_realloc: Option = None; let mut coreabi_sample_ids = Vec::new(); - t_stage = Instant::now(); for (id, expt) in module.exports.iter().enumerate() { match expt.name.as_str() { "coreabi_sample_i32" | "coreabi_sample_i64" | "coreabi_sample_f32" @@ -627,10 +435,6 @@ fn synthesize_import_functions( _ => {} }; } - eprintln!( - "trace(wasm-splice:imports:scan-exports): {} ms", - t_stage.elapsed().as_millis() - ); let memory = 0; @@ -697,7 +501,6 @@ fn synthesize_import_functions( // let mut import_fnids: Vec = Vec::new(); { - let t_loop = Instant::now(); // synthesized native import function parameters (in order) let ctx_arg = coreabi_sample_i32.args[0]; // let argc_arg = coreabi_sample_i32.args[1]; // Unused @@ -914,14 +717,8 @@ fn synthesize_import_functions( let fid = func.finish_module(module); import_fnids.push(fid); } - eprintln!( - "trace(wasm-splice:imports:build-fns): {} ms ({} imports)", - t_loop.elapsed().as_millis(), - imports.len() - ); // extend the main table to include indices for generated imported functions - t_stage = Instant::now(); let table = module.tables.get_mut(main_tid); table.initial += imports.len() as u64; table.maximum = Some(table.maximum.unwrap() + imports.len() as u64); @@ -933,10 +730,6 @@ fn synthesize_import_functions( funcs.push(fid); } } - eprintln!( - "trace(wasm-splice:imports:update-table): {} ms", - t_stage.elapsed().as_millis() - ); } // Populate the import creation function of the form: @@ -948,7 +741,6 @@ fn synthesize_import_functions( // } // { - t_stage = Instant::now(); let coreabi_get_import_fid = get_export_fid(module, &coreabi_get_import.unwrap()); let args = &module @@ -1007,36 +799,20 @@ fn synthesize_import_functions( .instructions .add_instr(table_instr_idx, Operator::I32Add); builder.body.instructions.finish_instr(table_instr_idx); - eprintln!( - "trace(wasm-splice:imports:fixup-get-import): {} ms", - t_stage.elapsed().as_millis() - ); } // remove unnecessary exports - t_stage = Instant::now(); module.exports.delete(coreabi_to_bigint64); module.exports.delete(coreabi_from_bigint64); module.exports.delete(coreabi_get_import.unwrap()); for id in coreabi_sample_ids { module.exports.delete(id); } - eprintln!( - "trace(wasm-splice:imports:prune-exports): {} ms", - t_stage.elapsed().as_millis() - ); - - eprintln!( - "trace(wasm-splice:imports:total): {} ms", - t_total.elapsed().as_millis() - ); Ok(()) } fn synthesize_export_functions(module: &mut Module, exports: &[(String, CoreFn)]) -> Result<()> { - let t_total = Instant::now(); - let mut t_stage = Instant::now(); let cabi_realloc = get_export_fid( module, &module @@ -1057,7 +833,6 @@ fn synthesize_export_functions(module: &mut Module, exports: &[(String, CoreFn)] let memory = 0; // (2) Export call function synthesis - let t_loop = Instant::now(); for (export_num, (expt_name, expt_sig)) in exports.iter().enumerate() { // Export function synthesis { @@ -1252,25 +1027,10 @@ fn synthesize_export_functions(module: &mut Module, exports: &[(String, CoreFn)] .exports .add_export_func(format!("cabi_post_{expt_name}"), *fid); } - eprintln!( - "trace(wasm-splice:exports:build-fns): {} ms ({} exports)", - t_loop.elapsed().as_millis(), - exports.len() - ); // remove unnecessary exports - t_stage = Instant::now(); module.exports.delete(call_expt); module.exports.delete(post_call_expt); - eprintln!( - "trace(wasm-splice:exports:prune-exports): {} ms", - t_stage.elapsed().as_millis() - ); - - eprintln!( - "trace(wasm-splice:exports:total): {} ms", - t_total.elapsed().as_millis() - ); Ok(()) } diff --git a/crates/spidermonkey-embedding-splicer/src/stub_wasi.rs b/crates/spidermonkey-embedding-splicer/src/stub_wasi.rs index e1159381..7484d405 100644 --- a/crates/spidermonkey-embedding-splicer/src/stub_wasi.rs +++ b/crates/spidermonkey-embedding-splicer/src/stub_wasi.rs @@ -120,9 +120,7 @@ pub fn stub_wasi( target_world_imports.insert(resolve.name_canonicalized_world_key(key)); } - let start_time = std::time::Instant::now(); let mut module = Module::parse(wasm.as_slice(), false).unwrap(); - println!("stub_wasi module parse took {:?}", start_time.elapsed()); stub_preview1(&mut module)?; diff --git a/src/componentize.js b/src/componentize.js index e20f4ae5..5a6c3383 100644 --- a/src/componentize.js +++ b/src/componentize.js @@ -298,8 +298,6 @@ export async function componentize( let postProcess; const wizerBin = opts.wizerBin ?? wizer; - console.error('trace(node:wizer): starting'); - const t_wizer_start = Date.now(); postProcess = spawnSync( wizerBin, [ @@ -320,10 +318,6 @@ export async function componentize( encoding: 'utf-8', }, ); - const t_wizer_end = Date.now(); - console.error( - `trace(node:wizer): ${(t_wizer_end - t_wizer_start)} ms (includes engine init + snapshot)`, - ); // If the wizer process failed, parse the output and display to the user if (postProcess.status !== 0) { @@ -338,20 +332,14 @@ export async function componentize( } // Read the generated WASM back into memory - const t_read_out_start = Date.now(); const bin = await readFile(outputWasmPath); - const t_read_out_end = Date.now(); - console.error( - `trace(node:read-out-wasm): ${(t_read_out_end - t_read_out_start)} ms`); // Check for initialization errors, by actually executing the binary in // a mini sandbox to get back the initialization state - const t_check_start = Date.now(); const { exports: { check_init }, getStderr, } = await initWasm(bin); - const t_check_mid = Date.now(); /// Process output of check init, throwing if necessary await handleCheckInitOutput( @@ -360,10 +348,6 @@ export async function componentize( workDir, getStderr, ); - const t_check_end = Date.now(); - console.error( - `trace(node:check-init): initWasm ${(t_check_mid - t_check_start)} ms, check_init ${(t_check_end - t_check_mid)} ms`, - ); // After wizening, stub out the wasi imports depending on what features are enabled let finalBin; @@ -399,7 +383,6 @@ export async function componentize( await writeFile('binary.wasm', finalBin); } - const t_component_start = Date.now(); const adapterBytes = await readFile(preview2Adapter); const rt = await componentNew( finalBin, @@ -408,7 +391,6 @@ export async function componentize( }), false, ); - const t_component_mid = Date.now(); const component = await metadataAdd( rt, Object.entries({ @@ -416,10 +398,6 @@ export async function componentize( 'processed-by': [['ComponentizeJS', version]], }), ); - const t_component_end = Date.now(); - console.error( - `trace(node:componentize): componentNew ${(t_component_mid - t_component_start)} ms, metadataAdd ${(t_component_end - t_component_mid)} ms`, - ); // Convert CABI import conventions to ESM import conventions imports = imports.map(([specifier, impt]) => diff --git a/src/splicer.js b/src/splicer.js index 2db5b587..aaf178c3 100644 --- a/src/splicer.js +++ b/src/splicer.js @@ -97,8 +97,6 @@ export async function spliceBindingsCli( workDir, opts = {}, ) { - const t_start = Date.now(); - // Prepare temporary directory for splicer output const splicerOutDir = join(workDir, 'splicer-out'); await mkdir(splicerOutDir); @@ -127,9 +125,6 @@ export async function spliceBindingsCli( const exports = JSON.parse(await readFile(join(splicerOutDir, 'exports.json'), 'utf8')); const imports = JSON.parse(await readFile(join(splicerOutDir, 'imports.json'), 'utf8')); - const t_end = Date.now(); - console.error(`trace(splicer-cli:splice-bindings): ${t_end - t_start} ms`); - return { wasm, jsBindings, exports, imports }; } @@ -151,8 +146,6 @@ export async function stubWasiCli( workDir, opts = {}, ) { - const t_start = Date.now(); - // Write wasm to temp file for stubbing const stubInputPath = join(workDir, 'stub-input.wasm'); const stubOutputPath = join(workDir, 'stub-output.wasm'); @@ -170,9 +163,6 @@ export async function stubWasiCli( // Read the stubbed wasm const finalBin = await readFile(stubOutputPath); - const t_end = Date.now(); - console.error(`trace(splicer-cli:stub-wasi): ${t_end - t_start} ms`); - return finalBin; } @@ -194,8 +184,6 @@ export async function spliceBindingsWasm( worldName, debug, ) { - const t_start = Date.now(); - const result = wasmSplicer.spliceBindings( engineWasm, features, @@ -205,9 +193,6 @@ export async function spliceBindingsWasm( debug, ); - const t_end = Date.now(); - console.error(`trace(splicer-wasm:splice-bindings): ${t_end - t_start} ms`); - return { wasm: Buffer.from(result.wasm), jsBindings: result.jsBindings, @@ -232,8 +217,6 @@ export async function stubWasiWasm( witPath, worldName, ) { - const t_start = Date.now(); - const result = wasmSplicer.stubWasi( wasmBinary, features, @@ -242,9 +225,6 @@ export async function stubWasiWasm( worldName, ); - const t_end = Date.now(); - console.error(`trace(splicer-wasm:stub-wasi): ${t_end - t_start} ms`); - return Buffer.from(result); } From b99befd7e7c4d0067b3f62937a086c21d9cd9e18 Mon Sep 17 00:00:00 2001 From: EngoDev Date: Tue, 7 Oct 2025 19:12:33 +0300 Subject: [PATCH 6/6] Formatted the code --- Cargo.toml | 1 - .../src/bin/splicer.rs | 12 ++++++------ crates/spidermonkey-embedding-splicer/src/splice.rs | 2 ++ 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 95c764e6..80a49c0c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,6 @@ wasmparser = { version = "0.239.0", features = ["features", "std", "validate", ] } - wit-bindgen = { version = "0.41.0", features = [ "macros", "async", "realloc" ] } wit-bindgen-core = { version = "0.41.0", default-features = false } wit-component = { version = "0.227.1", features = ["dummy-module"] } diff --git a/crates/spidermonkey-embedding-splicer/src/bin/splicer.rs b/crates/spidermonkey-embedding-splicer/src/bin/splicer.rs index fb53dc8d..09124898 100644 --- a/crates/spidermonkey-embedding-splicer/src/bin/splicer.rs +++ b/crates/spidermonkey-embedding-splicer/src/bin/splicer.rs @@ -208,7 +208,7 @@ fn serialize_imports(imports: &[(String, String, u32)]) -> String { /// Manually serialize CoreFn to JSON fn serialize_core_fn(core_fn: &CoreFn) -> String { let mut result = String::from("{"); - + // params result.push_str("\"params\": ["); for (i, param) in core_fn.params.iter().enumerate() { @@ -218,7 +218,7 @@ fn serialize_core_fn(core_fn: &CoreFn) -> String { result.push_str(&serialize_core_ty(param)); } result.push_str("], "); - + // ret result.push_str("\"ret\": "); if let Some(ref ret) = core_fn.ret { @@ -227,16 +227,16 @@ fn serialize_core_fn(core_fn: &CoreFn) -> String { result.push_str("null"); } result.push_str(", "); - + // retptr result.push_str(&format!("\"retptr\": {}, ", core_fn.retptr)); - + // retsize result.push_str(&format!("\"retsize\": {}, ", core_fn.retsize)); - + // paramptr result.push_str(&format!("\"paramptr\": {}", core_fn.paramptr)); - + result.push('}'); result } diff --git a/crates/spidermonkey-embedding-splicer/src/splice.rs b/crates/spidermonkey-embedding-splicer/src/splice.rs index 569ac805..039d573a 100644 --- a/crates/spidermonkey-embedding-splicer/src/splice.rs +++ b/crates/spidermonkey-embedding-splicer/src/splice.rs @@ -764,6 +764,8 @@ fn synthesize_import_functions( let ops_ro = builder.body.instructions.get_ops(); for (idx, op) in ops_ro.iter().enumerate() { if let Operator::I32Const { value } = op { + // we specifically need the const "around" 3393 + // which is the coreabi_sample_i32 table offset if *value < 1000 || *value > 5000 { continue; }