feat: router-based wasm bundle splitting (#3683)
* feat: wasm-split base implementation * kinda sorta works with the cli * okay great it actually works * fix: cache busting for chunks * cleaned up version * it works with the router * wip: migrate to a faster linear solution using id remapping * add a good harness * it works completely on the docsite * the super tight build works too * make parallelizable * works on the docsite but not the harness * light cleanups, fix for both harness and docsite * make codegen smaller by extract some shared functions to prevent monos * chunking works, but tbd on tuning heuristics * extract used module * wip: cleaning up module * wip: much better understanding of how imports work * fix the call graph * working again... * wip: thinking about ifuncs * ifunc approach works * yes the ifunc approach works!! * it all works completely even with chunks * small cleanups * comments * actually locate all of the symbols * clean up final implementation * clean up tomls, revert some mono changs * clean up, add some more comments / docs in places * hoist more packages * clean up impl a bit more * clean cli * move harness to playwright * update playwright * extract codgen * add lazy to components * Add docs to split loader * clippy * typos * clippy, don't generate splits if not on wasm * whoops, const * Typos and docs * enable optimizations in ci * add optimizations to ci * add wasm-split router feature * parallel playwright? * fix no case check * disable pre-compress... think that might be the issue * better logging, no default pre-compress, caching of shared symbols * Fix: compressing assets * fix non-interactive logging for serve * cache playwright even on failure, use the cli with optimizations * fix prebuild * fix playwright webserver
This commit is contained in:
Jonathan Kelley authored and GitHub committed
1 parent
14eb7597df
commit
bdeedc13eb
59 files changed
+4249
-299
No files matched your search
@@ -0,0 +1,63 @@
|
||||
// when running the harness we need to make sure to uncommon this out...
|
||||
|
||||
export function makeLoad(url, deps, fusedImports, initIt) {
|
||||
let alreadyLoaded = false;
|
||||
return async (callbackIndex, callbackData) => {
|
||||
await Promise.all(deps.map((dep) => dep()));
|
||||
if (alreadyLoaded) return;
|
||||
try {
|
||||
const response = await fetch(url);
|
||||
const initSync = initIt || window.__wasm_split_main_initSync;
|
||||
const mainExports = initSync(undefined, undefined);
|
||||
|
||||
let imports = {
|
||||
env: {
|
||||
memory: mainExports.memory,
|
||||
},
|
||||
__wasm_split: {
|
||||
__indirect_function_table: mainExports.__indirect_function_table,
|
||||
__stack_pointer: mainExports.__stack_pointer,
|
||||
__tls_base: mainExports.__tls_base,
|
||||
memory: mainExports.memory,
|
||||
},
|
||||
};
|
||||
|
||||
for (let mainExport in mainExports) {
|
||||
imports["__wasm_split"][mainExport] = mainExports[mainExport];
|
||||
}
|
||||
|
||||
for (let name in fusedImports) {
|
||||
imports["__wasm_split"][name] = fusedImports[name];
|
||||
}
|
||||
|
||||
let new_exports = await WebAssembly.instantiateStreaming(
|
||||
response,
|
||||
imports
|
||||
);
|
||||
|
||||
alreadyLoaded = true;
|
||||
|
||||
for (let name in new_exports.instance.exports) {
|
||||
fusedImports[name] = new_exports.instance.exports[name];
|
||||
}
|
||||
|
||||
if (callbackIndex !== undefined) {
|
||||
mainExports.__indirect_function_table.get(callbackIndex)(
|
||||
callbackData,
|
||||
true
|
||||
);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(
|
||||
"Failed to load wasm-split module",
|
||||
e,
|
||||
url,
|
||||
deps,
|
||||
fusedImports
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
let fusedImports = {};
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,178 @@
|
||||
use clap::Parser;
|
||||
use std::path::PathBuf;
|
||||
use wasm_split_cli::SplitModule;
|
||||
|
||||
fn main() {
|
||||
tracing_subscriber::fmt()
|
||||
.without_time()
|
||||
.compact()
|
||||
.with_env_filter("debug,walrus=info")
|
||||
.init();
|
||||
|
||||
match Commands::parse() {
|
||||
Commands::Split(split_args) => split(split_args),
|
||||
Commands::Validate(validate_args) => validate(validate_args),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Parser)]
|
||||
enum Commands {
|
||||
/// Split a wasm module into multiple chunks
|
||||
#[clap(name = "split")]
|
||||
Split(SplitArgs),
|
||||
|
||||
/// Validate the main module of a wasm module
|
||||
#[clap(name = "validate")]
|
||||
Validate(ValidateArgs),
|
||||
}
|
||||
|
||||
#[derive(Parser)]
|
||||
struct SplitArgs {
|
||||
/// The wasm module emitted by rustc
|
||||
original: PathBuf,
|
||||
|
||||
/// The wasm module emitted by wasm-bindgen
|
||||
bindgened: PathBuf,
|
||||
|
||||
/// The output *directory* to write the split wasm files to
|
||||
out_dir: PathBuf,
|
||||
}
|
||||
|
||||
fn split(args: SplitArgs) {
|
||||
let original = std::fs::read(&args.original).expect("failed to read input file");
|
||||
let bindgened = std::fs::read(&args.bindgened).expect("failed to read input file");
|
||||
|
||||
_ = std::fs::remove_dir_all(&args.out_dir);
|
||||
std::fs::create_dir_all(&args.out_dir).expect("failed to create output dir");
|
||||
|
||||
tracing::info!("Building split module");
|
||||
|
||||
let module = wasm_split_cli::Splitter::new(&original, &bindgened).unwrap();
|
||||
|
||||
let mut chunks = module.emit().unwrap();
|
||||
|
||||
// Write out the main module
|
||||
tracing::info!(
|
||||
"Writing main module to {}",
|
||||
args.out_dir.join("main.wasm").display()
|
||||
);
|
||||
std::fs::write(args.out_dir.join("main.wasm"), &chunks.main.bytes).unwrap();
|
||||
|
||||
// Write the js module
|
||||
std::fs::write(
|
||||
args.out_dir.join("__wasm_split.js"),
|
||||
emit_js(&chunks.chunks, &chunks.modules),
|
||||
)
|
||||
.expect("failed to write js module");
|
||||
|
||||
for (idx, chunk) in chunks.chunks.iter().enumerate() {
|
||||
tracing::info!(
|
||||
"Writing chunk {} to {}",
|
||||
idx,
|
||||
args.out_dir
|
||||
.join(format!("chunk_{}_{}.wasm", idx, chunk.module_name))
|
||||
.display()
|
||||
);
|
||||
std::fs::write(
|
||||
args.out_dir
|
||||
.join(format!("chunk_{}_{}.wasm", idx, chunk.module_name)),
|
||||
&chunk.bytes,
|
||||
)
|
||||
.expect("failed to write chunk");
|
||||
}
|
||||
|
||||
for (idx, module) in chunks.modules.iter_mut().enumerate() {
|
||||
tracing::info!(
|
||||
"Writing module {} to {}",
|
||||
idx,
|
||||
args.out_dir
|
||||
.join(format!(
|
||||
"module_{}_{}.wasm",
|
||||
idx,
|
||||
module.component_name.as_ref().unwrap()
|
||||
))
|
||||
.display()
|
||||
);
|
||||
std::fs::write(
|
||||
args.out_dir.join(format!(
|
||||
"module_{}_{}.wasm",
|
||||
idx,
|
||||
module.component_name.as_ref().unwrap()
|
||||
)),
|
||||
&module.bytes,
|
||||
)
|
||||
.expect("failed to write chunk");
|
||||
}
|
||||
}
|
||||
|
||||
fn emit_js(chunks: &[SplitModule], modules: &[SplitModule]) -> String {
|
||||
use std::fmt::Write;
|
||||
let mut glue = format!(
|
||||
r#"import {{ initSync }} from "./main.js";
|
||||
{}"#,
|
||||
include_str!("./__wasm_split.js")
|
||||
);
|
||||
|
||||
for (idx, chunk) in chunks.iter().enumerate() {
|
||||
tracing::debug!("emitting chunk: {:?}", chunk.module_name);
|
||||
writeln!(
|
||||
glue,
|
||||
"export const __wasm_split_load_chunk_{idx} = makeLoad(\"/harness/split/chunk_{idx}_{module}.wasm\", [], fusedImports, initSync);",
|
||||
module = chunk.module_name
|
||||
).expect("failed to write to string");
|
||||
}
|
||||
|
||||
// Now write the modules
|
||||
for (idx, module) in modules.iter().enumerate() {
|
||||
let deps = module
|
||||
.relies_on_chunks
|
||||
.iter()
|
||||
.map(|idx| format!("__wasm_split_load_chunk_{idx}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
let hash_id = module.hash_id.as_ref().unwrap();
|
||||
|
||||
writeln!(
|
||||
glue,
|
||||
"export const __wasm_split_load_{module}_{hash_id}_{cname} = makeLoad(\"/harness/split/module_{idx}_{cname}.wasm\", [{deps}], fusedImports, initSync);",
|
||||
module = module.module_name,
|
||||
idx = idx,
|
||||
cname = module.component_name.as_ref().unwrap(),
|
||||
deps = deps
|
||||
)
|
||||
.expect("failed to write to string");
|
||||
}
|
||||
|
||||
glue
|
||||
}
|
||||
|
||||
#[derive(Parser)]
|
||||
struct ValidateArgs {
|
||||
/// The input wasm file to validate
|
||||
main: PathBuf,
|
||||
|
||||
chunks: Vec<PathBuf>,
|
||||
}
|
||||
|
||||
fn validate(args: ValidateArgs) {
|
||||
let bytes = std::fs::read(&args.main).expect("failed to read input file");
|
||||
let main_module = walrus::Module::from_buffer(&bytes).unwrap();
|
||||
|
||||
for chunk in args.chunks {
|
||||
let bytes = std::fs::read(chunk).expect("failed to read input file");
|
||||
let chunk_module = walrus::Module::from_buffer(&bytes).unwrap();
|
||||
|
||||
assert!(chunk_module.tables.iter().count() == 1);
|
||||
|
||||
for import in chunk_module.imports.iter() {
|
||||
let matching = main_module.exports.iter().find(|e| e.name == import.name);
|
||||
|
||||
let Some(matching) = matching else {
|
||||
tracing::error!("Could not find matching export for import {import:#?}");
|
||||
continue;
|
||||
};
|
||||
|
||||
tracing::debug!("import: {:?}", matching.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user