Make wasm-bindgen parallel/disable log, speed up 2-10x (#3161)

* wip: pull out incr linker intercept and debug=true for wasm for faster builds

* make walrus parallel

* clean up impl

Cargo.toml

@@ -245,6 +245,11 @@ tray-icon = "0.19"
 incremental = true
 debug = 0
 
+[profile.release-max-opt]
+inherits = "release"
+lto = true
+codegen-units = 1
+
 # Disable debug assertions to check the released path of core and other packages, but build without optimizations to keep build times quick
 [profile.release-unoptimized]
 inherits = "dev"

packages/cli/Cargo.toml

@@ -61,6 +61,7 @@ syn = { workspace = true, features = ["full", "extra-traits", "visit", "visit-mu
 
 headers = "0.4.0"
 walkdir = "2"
+walrus = { workspace = true, features = ["parallel"] }
 
 # tools download
 dirs = { workspace = true }
@@ -99,6 +100,9 @@ ansi-to-tui = "6.0"
 ansi-to-html = "0.2.1"
 ratatui = { version = "0.28.0", features = ["crossterm", "unstable"] }
 
+# disable `log` entirely since `walrus` uses it and is *much* slower with it enableda
+log = { version = "0.4", features = ["max_level_off", "release_max_level_off"] }
+
 # link intercept
 tempfile = "3.3"
 manganis-core = { workspace = true }

packages/cli/src/assets.rs

@@ -15,6 +15,7 @@ pub(crate) struct AssetManifest {
 }
 
 impl AssetManifest {
+    #[allow(dead_code)]
     pub(crate) fn load_from_file(path: &Path) -> anyhow::Result<Self> {
         let src = std::fs::read_to_string(path)
             .context("Failed to read asset manifest from filesystem")?;
@@ -24,31 +25,20 @@ impl AssetManifest {
 
     /// Fill this manifest with a file object/rlib files, typically extracted from the linker intercepted
     pub(crate) fn add_from_object_path(&mut self, path: PathBuf) -> anyhow::Result<()> {
-        let Some(ext) = path.extension() else {
-            return Ok(());
-        };
-
-        let Some(ext) = ext.to_str() else {
-            return Ok(());
-        };
-
         let data = std::fs::read(path.clone())?;
 
-        match ext {
+        match path.extension().and_then(|ext| ext.to_str()) {
-            // Parse an unarchived object file
-            "o" => {
-                if let Ok(object) = object::File::parse(&*data) {
-                    self.add_from_object_file(&object)?;
-                }
-            }
-
             // Parse an rlib as a collection of objects
-            "rlib" => {
+            Some("rlib") => {
                 if let Ok(archive) = object::read::archive::ArchiveFile::parse(&*data) {
                     self.add_from_archive_file(&archive, &data)?;
                 }
             }
-            _ => {}
+            _ => {
+                if let Ok(object) = object::File::parse(&*data) {
+                    self.add_from_object_file(&object)?;
+                }
+            }
         }
 
         Ok(())

packages/cli/src/builder/build.rs

@@ -6,7 +6,10 @@ use crate::{build::BuildArgs, link::LinkAction};
 use crate::{AppBundle, Platform};
 use anyhow::Context;
 use serde::Deserialize;
-use std::{path::PathBuf, process::Stdio};
+use std::{
+    path::{Path, PathBuf},
+    process::Stdio,
+};
 use tokio::{io::AsyncBufReadExt, process::Command};
 
 #[derive(Clone, Debug)]
@@ -50,7 +53,7 @@ impl BuildRequest {
     pub(crate) async fn build_app(&self) -> Result<BuildArtifacts> {
         tracing::debug!("Building app...");
         let exe = self.build_cargo().await?;
-        let assets = self.collect_assets().await?;
+        let assets = self.collect_assets(&exe).await?;
         Ok(BuildArtifacts { exe, assets })
     }
 
@@ -185,7 +188,7 @@ impl BuildRequest {
     /// This will execute `dx` with an env var set to force `dx` to operate as a linker, and then
     /// traverse the .o and .rlib files rustc passes that new `dx` instance, collecting the link
     /// tables marked by manganis and parsing them as a ResourceAsset.
-    pub(crate) async fn collect_assets(&self) -> anyhow::Result<AssetManifest> {
+    pub(crate) async fn collect_assets(&self, exe: &Path) -> Result<AssetManifest> {
         tracing::debug!("Collecting assets ...");
 
         // If assets are skipped, we don't need to collect them
@@ -193,43 +196,11 @@ impl BuildRequest {
             return Ok(AssetManifest::default());
         }
 
-        // Create a temp file to put the output of the args
+        let mut manifest = AssetManifest::default();
-        // We need to do this since rustc won't actually print the link args to stdout, so we need to
-        // give `dx` a file to dump its env::args into
-        let tmp_file = tempfile::NamedTempFile::new()?;
 
-        // Run `cargo rustc` again, but this time with a custom linker (dx) and an env var to force
+        _ = manifest.add_from_object_path(exe.to_path_buf());
-        // `dx` to act as a linker
-        //
-        // This will force `dx` to look through the incremental cache and find the assets from the previous build
-        Command::new("cargo")
-            // .env("RUSTFLAGS", self.rust_flags())
-            .arg("rustc")
-            .args(self.build_arguments())
-            .arg("--offline") /* don't use the network, should already be resolved */
-            .arg("--")
-            .arg(format!(
-                "-Clinker={}",
-                std::env::current_exe()
-                    .unwrap()
-                    .canonicalize()
-                    .unwrap()
-                    .display()
-            ))
-            .env(
-                LinkAction::ENV_VAR_NAME,
-                LinkAction::BuildAssetManifest {
-                    destination: tmp_file.path().to_path_buf(),
-                }
-                .to_json(),
-            )
-            .stdout(Stdio::piped())
-            .stderr(Stdio::piped())
-            .output()
-            .await?;
 
-        // The linker wrote the manifest to the temp file, let's load it!
+        Ok(manifest)
-        AssetManifest::load_from_file(tmp_file.path())
     }
 
     /// Create a list of arguments for cargo builds
@@ -416,4 +387,51 @@ impl BuildRequest {
                 / 3.5) as usize
         })
     }
+
+    /// We used to require traversing incremental artifacts for assets that were included but not
+    /// directly exposed to the final binary. Now, however, we force APIs to carry items created
+    /// from asset calls into top-level items such that they *do* get included in the final binary.
+    ///
+    /// There's a chance that's not actually true, so this function is kept around in case we do
+    /// need to revert to "deep extraction".
+    #[allow(unused)]
+    async fn deep_linker_asset_extract(&self) -> Result<AssetManifest> {
+        // Create a temp file to put the output of the args
+        // We need to do this since rustc won't actually print the link args to stdout, so we need to
+        // give `dx` a file to dump its env::args into
+        let tmp_file = tempfile::NamedTempFile::new()?;
+
+        // Run `cargo rustc` again, but this time with a custom linker (dx) and an env var to force
+        // `dx` to act as a linker
+        //
+        // This will force `dx` to look through the incremental cache and find the assets from the previous build
+        Command::new("cargo")
+            // .env("RUSTFLAGS", self.rust_flags())
+            .arg("rustc")
+            .args(self.build_arguments())
+            .arg("--offline") /* don't use the network, should already be resolved */
+            .arg("--")
+            .arg(format!(
+                "-Clinker={}",
+                std::env::current_exe()
+                    .unwrap()
+                    .canonicalize()
+                    .unwrap()
+                    .display()
+            ))
+            .env(
+                LinkAction::ENV_VAR_NAME,
+                LinkAction::BuildAssetManifest {
+                    destination: tmp_file.path().to_path_buf(),
+                }
+                .to_json(),
+            )
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .output()
+            .await?;
+
+        // The linker wrote the manifest to the temp file, let's load it!
+        Ok(AssetManifest::load_from_file(tmp_file.path())?)
+    }
 }

packages/cli/src/builder/bundle.rs

@@ -653,7 +653,8 @@ impl AppBundle {
         let input_path = input_path.to_path_buf();
         let bindgen_outdir = bindgen_outdir.to_path_buf();
         let name = self.build.krate.executable_name().to_string();
-        let keep_debug = self.build.krate.config.web.wasm_opt.debug || (!self.build.build.release);
+        let keep_debug = self.build.krate.config.web.wasm_opt.debug;
+        let reference_types = self.build.krate.config.web.wasm_opt.reference_types;
 
         let start = std::time::Instant::now();
         tokio::task::spawn_blocking(move || {
@@ -664,9 +665,9 @@ impl AppBundle {
                 .debug(keep_debug)
                 .demangle(keep_debug)
                 .keep_debug(keep_debug)
-                .reference_types(true)
+                .reference_types(reference_types)
-                .remove_name_section(!keep_debug)
+                .remove_name_section(true)
-                .remove_producers_section(!keep_debug)
+                .remove_producers_section(true)
                 .out_name(&name)
                 .generate(&bindgen_outdir)
         })

packages/cli/src/config/web.rs

@@ -58,6 +58,10 @@ pub(crate) struct WasmOptConfig {
     /// Keep debug symbols in the wasm file
     #[serde(default = "false_bool")]
     pub(crate) debug: bool,
+
+    /// Enable reference types
+    #[serde(default = "false_bool")]
+    pub(crate) reference_types: bool,
 }
 
 /// The wasm-opt level to use for release web builds [default: 4]