Optimize wasm bindgen asset with manganis (#3531)

packages/cli-opt/Cargo.toml

@@ -39,57 +39,30 @@ grass = "0.13.4"
 codemap = "0.1.3"
 
 # Js minification - swc has introduces minor versions with breaking changes in the past so we pin all of their crates
-swc = "=0.285.0"
-swc_allocator = { version = "=0.1.8", default-features = false }
-swc_atoms = { version = "=0.6.7", default-features = false }
-swc_cached = { version = "=0.3.20", default-features = false }
-swc_common = { version = "=0.37.5", default-features = false }
-swc_compiler_base = { version = "=0.19.0", default-features = false }
-swc_config = { version = "=0.1.15", default-features = false }
-swc_config_macro = { version = "=0.1.4", default-features = false }
-swc_ecma_ast = { version = "=0.118.2", default-features = false }
-swc_ecma_codegen = { version = "=0.155.1", default-features = false }
-swc_ecma_codegen_macros = { version = "=0.7.7", default-features = false }
-swc_ecma_compat_bugfixes = { version = "=0.12.0", default-features = false }
-swc_ecma_compat_common = { version = "=0.11.0", default-features = false }
-swc_ecma_compat_es2015 = { version = "=0.12.0", default-features = false }
-swc_ecma_compat_es2016 = { version = "=0.12.0", default-features = false }
-swc_ecma_compat_es2017 = { version = "=0.12.0", default-features = false }
-swc_ecma_compat_es2018 = { version = "=0.12.0", default-features = false }
-swc_ecma_compat_es2019 = { version = "=0.12.0", default-features = false }
-swc_ecma_compat_es2020 = { version = "=0.12.0", default-features = false }
-swc_ecma_compat_es2021 = { version = "=0.12.0", default-features = false }
-swc_ecma_compat_es2022 = { version = "=0.12.0", default-features = false }
-swc_ecma_compat_es3 = { version = "=0.12.0", default-features = false }
-swc_ecma_ext_transforms = { version = "=0.120.0", default-features = false }
-swc_ecma_lints = { version = "=0.100.0", default-features = false }
-swc_ecma_loader = { version = "=0.49.1", default-features = false }
-swc_ecma_minifier = { version = "=0.204.0", default-features = false }
-swc_ecma_parser = { version = "=0.149.1", default-features = false }
-swc_ecma_preset_env = { version = "=0.217.0", default-features = false, features = [
-    "serde",
-] }
-swc_ecma_transforms = { version = "=0.239.0", default-features = false }
-swc_ecma_transforms_base = { version = "=0.145.0", default-features = false }
-swc_ecma_transforms_classes = { version = "=0.134.0", default-features = false }
-swc_ecma_transforms_compat = { version = "=0.171.0", default-features = false }
-swc_ecma_transforms_macros = { version = "=0.5.5", default-features = false }
-swc_ecma_transforms_module = { version = "=0.190.0", default-features = false }
-swc_ecma_transforms_optimization = { version = "=0.208.0", default-features = false }
-swc_ecma_transforms_proposal = { version = "=0.178.0", default-features = false }
-swc_ecma_transforms_react = { version = "=0.191.0", default-features = false }
-swc_ecma_transforms_typescript = { version = "=0.198.1", default-features = false }
-swc_ecma_usage_analyzer = { version = "=0.30.3", default-features = false }
-swc_ecma_utils = { version = "=0.134.2", default-features = false }
-swc_ecma_visit = { version = "=0.104.8", default-features = false }
-swc_eq_ignore_macros = { version = "=0.1.4", default-features = false }
-swc_error_reporters = { version = "=0.21.0", default-features = false }
-swc_fast_graph = { version = "=0.25.0", default-features = false }
-swc_macros_common = { version = "=0.3.13", default-features = false }
-swc_node_comments = { version = "=0.24.0", default-features = false }
-swc_timer = { version = "=0.25.0", default-features = false }
-swc_trace_macro = { version = "=0.1.3", default-features = false }
-swc_transform_common = { version = "=0.1.1", default-features = false }
-swc_typescript = { version = "=0.5.0", default-features = false }
-swc_visit = { version = "=0.6.2", default-features = false }
-browserslist-rs = { version = "=0.16.0" }
+swc_allocator = { version = "=2.0.0", default-features = false }
+swc_atoms = { version = "=3.0.2", default-features = false }
+swc_bundler = { version = "=7.0.0", default-features = false }
+swc_cached = { version = "=1.0.0", default-features = false }
+swc_common = { version = "=5.0.0", features = ["tty-emitter"], default-features = false }
+swc_config = { version = "=1.0.0", default-features = false }
+swc_config_macro = { version = "=1.0.0", default-features = false }
+swc_ecma_ast = { version = "=5.0.1", default-features = false }
+swc_ecma_codegen = { version = "=5.0.1", default-features = false }
+swc_ecma_codegen_macros = { version = "=1.0.0", default-features = false }
+swc_ecma_loader = { version = "=5.0.0", features = ["cache", "node"], default-features = false }
+swc_ecma_minifier = { version = "=7.0.1", default-features = false }
+swc_ecma_parser = { version = "=6.0.2", default-features = false }
+swc_ecma_transforms_base = { version = "=7.0.0", default-features = false }
+swc_ecma_transforms_macros = { version = "=1.0.0", default-features = false }
+swc_ecma_transforms_optimization = { version = "=7.0.1", default-features = false }
+swc_ecma_usage_analyzer = { version = "=7.0.0", default-features = false }
+swc_ecma_utils = { version = "=7.0.0", default-features = false }
+swc_ecma_visit = { version = "=5.0.0", default-features = false }
+swc_eq_ignore_macros = { version = "=1.0.0", default-features = false }
+swc_fast_graph = { version = "=6.0.0", default-features = false }
+swc_graph_analyzer = { version = "=5.0.0", default-features = false }
+swc_macros_common = { version = "=1.0.0", default-features = false }
+swc_parallel = { version = "=1.0.1", default-features = false }
+swc_timer = { version = "=1.0.0", default-features = false }
+swc_visit = { version = "=2.0.0", default-features = false }
+browserslist-rs = { version = "=0.16.0" }

packages/cli-opt/src/file.rs

@@ -1,6 +1,6 @@
 use anyhow::Context;
 use manganis_core::{AssetOptions, CssAssetOptions, ImageAssetOptions, JsAssetOptions};
-use std::path::Path;
+use std::{ffi::OsStr, path::Path};
 
 use crate::css::process_scss;
 
@@ -14,6 +14,16 @@ pub fn process_file_to(
     options: &AssetOptions,
     source: &Path,
     output_path: &Path,
+) -> anyhow::Result<()> {
+    process_file_to_with_options(options, source, output_path, false)
+}
+
+/// Process a specific file asset with additional options
+pub(crate) fn process_file_to_with_options(
+    options: &AssetOptions,
+    source: &Path,
+    output_path: &Path,
+    in_folder: bool,
 ) -> anyhow::Result<()> {
     // If the file already exists, then we must have a file with the same hash
     // already. The hash has the file contents and options, so if we find a file
@@ -26,56 +36,68 @@ pub fn process_file_to(
             std::fs::create_dir_all(parent)?;
         }
     }
+
+    // Processing can be slow. Write to a temporary file first and then rename it to the final output path. If everything
+    // goes well. Without this, the user could quit in the middle of processing and the file will look complete to the
+    // caching system even though it is empty.
+    let mut partial_ext = output_path.extension().unwrap_or_default().to_os_string();
+    partial_ext.push(OsStr::new(".partial"));
+    let temp_output_path = output_path.with_extension(&partial_ext);
+    let temp_output_path = &temp_output_path;
+
     match options {
         AssetOptions::Unknown => match source.extension().map(|e| e.to_string_lossy()).as_deref() {
             Some("css") => {
-                process_css(&CssAssetOptions::new(), source, output_path)?;
+                process_css(&CssAssetOptions::new(), source, temp_output_path)?;
             }
             Some("scss" | "sass") => {
-                process_scss(&CssAssetOptions::new(), source, output_path)?;
+                process_scss(&CssAssetOptions::new(), source, temp_output_path)?;
             }
             Some("js") => {
-                process_js(&JsAssetOptions::new(), source, output_path)?;
+                process_js(&JsAssetOptions::new(), source, temp_output_path, !in_folder)?;
             }
             Some("json") => {
-                process_json(source, output_path)?;
+                process_json(source, temp_output_path)?;
             }
             Some("jpg" | "jpeg" | "png" | "webp" | "avif") => {
-                process_image(&ImageAssetOptions::new(), source, output_path)?;
+                process_image(&ImageAssetOptions::new(), source, temp_output_path)?;
             }
             Some(_) | None => {
                 if source.is_dir() {
-                    process_folder(source, output_path)?;
+                    process_folder(source, temp_output_path)?;
                 } else {
                     let source_file = std::fs::File::open(source)?;
                     let mut reader = std::io::BufReader::new(source_file);
-                    let output_file = std::fs::File::create(output_path)?;
+                    let output_file = std::fs::File::create(temp_output_path)?;
                     let mut writer = std::io::BufWriter::new(output_file);
                     std::io::copy(&mut reader, &mut writer).with_context(|| {
                         format!(
                             "Failed to write file to output location: {}",
-                            output_path.display()
+                            temp_output_path.display()
                         )
                     })?;
                 }
             }
         },
         AssetOptions::Css(options) => {
-            process_css(options, source, output_path)?;
+            process_css(options, source, temp_output_path)?;
         }
         AssetOptions::Js(options) => {
-            process_js(options, source, output_path)?;
+            process_js(options, source, temp_output_path, !in_folder)?;
         }
         AssetOptions::Image(options) => {
-            process_image(options, source, output_path)?;
+            process_image(options, source, temp_output_path)?;
         }
         AssetOptions::Folder(_) => {
-            process_folder(source, output_path)?;
+            process_folder(source, temp_output_path)?;
         }
         _ => {
             tracing::warn!("Unknown asset options: {:?}", options);
         }
     }
 
+    // If everything was successful, rename the temp file to the final output path
+    std::fs::rename(temp_output_path, output_path)?;
+
     Ok(())
 }

packages/cli-opt/src/folder.rs

@@ -2,7 +2,7 @@ use std::path::Path;
 
 use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
 
-use super::file::process_file_to;
+use crate::file::process_file_to_with_options;
 
 /// Process a folder, optimizing and copying all assets into the output folder
 pub fn process_folder(source: &Path, output_folder: &Path) -> anyhow::Result<()> {
@@ -32,10 +32,11 @@ pub fn process_folder(source: &Path, output_folder: &Path) -> anyhow::Result<()>
 
 /// Optimize a file without changing any of its contents significantly (e.g. by changing the extension)
 fn process_file_minimal(input_path: &Path, output_path: &Path) -> anyhow::Result<()> {
-    process_file_to(
+    process_file_to_with_options(
         &manganis_core::AssetOptions::Unknown,
         input_path,
         output_path,
+        true,
     )?;
     Ok(())
 }

packages/cli-opt/src/js.rs

@@ -1,74 +1,243 @@
-use std::io::Read;
 use std::path::Path;
-use std::sync::Arc;
+use std::path::PathBuf;
 
 use anyhow::Context;
 use manganis_core::JsAssetOptions;
-use swc::{config::JsMinifyOptions, try_with_handler, BoolOrDataConfig, JsMinifyExtras};
-use swc_common::{sync::Lrc, FileName};
-use swc_common::{SourceMap, GLOBALS};
-use swc_ecma_minifier::option::MangleOptions;
+use swc_common::errors::Emitter;
+use swc_common::errors::Handler;
+use swc_common::input::SourceFileInput;
+use swc_ecma_minifier::option::{ExtraOptions, MinifyOptions};
+use swc_ecma_parser::lexer::Lexer;
+use swc_ecma_parser::Parser;
+use swc_ecma_transforms_base::fixer::fixer;
+use swc_ecma_visit::VisitMutWith;
 
-pub(crate) fn minify_js(source: &Path) -> anyhow::Result<String> {
-    let mut source_file = std::fs::File::open(source)?;
-    let cm = Arc::new(SourceMap::default());
+use std::collections::HashMap;
 
-    let mut js = String::new();
-    source_file.read_to_string(&mut js)?;
-    let c = swc::Compiler::new(cm.clone());
-    let output = GLOBALS
-        .set(&Default::default(), || {
-            try_with_handler(cm.clone(), Default::default(), |handler| {
-                let filename = Lrc::new(FileName::Real(source.to_path_buf()));
-                let fm = cm.new_source_file(filename, js.to_string());
+use anyhow::Error;
+use swc_bundler::{Bundler, Config, Load, ModuleData, ModuleRecord};
+use swc_common::{
+    errors::HANDLER, sync::Lrc, FileName, FilePathMapping, Globals, Mark, SourceMap, Span, GLOBALS,
+};
+use swc_ecma_ast::*;
+use swc_ecma_codegen::text_writer::JsWriter;
+use swc_ecma_loader::{resolvers::node::NodeModulesResolver, TargetEnv};
+use swc_ecma_parser::{parse_file_as_module, Syntax};
 
-                c.minify(
-                    fm,
-                    handler,
-                    &JsMinifyOptions {
-                        compress: BoolOrDataConfig::from_bool(true),
-                        mangle: BoolOrDataConfig::from_obj(MangleOptions {
-                            keep_fn_names: true,
-                            ..Default::default()
-                        }),
-                        ..Default::default()
-                    },
-                    JsMinifyExtras::default(),
-                )
-                .context("failed to minify javascript")
-            })
-        })
-        .map(|output| output.code);
+struct TracingEmitter;
 
-    match output {
-        Ok(output) => Ok(output),
-        Err(err) => {
-            tracing::error!("Failed to minify javascript: {}", err);
-            Ok(js)
+impl Emitter for TracingEmitter {
+    fn emit(&mut self, db: &swc_common::errors::DiagnosticBuilder<'_>) {
+        match db.level {
+            swc_common::errors::Level::Bug
+            | swc_common::errors::Level::Fatal
+            | swc_common::errors::Level::PhaseFatal
+            | swc_common::errors::Level::Error => tracing::error!("{}", db.message()),
+            swc_common::errors::Level::Warning
+            | swc_common::errors::Level::FailureNote
+            | swc_common::errors::Level::Cancelled => tracing::warn!("{}", db.message()),
+            swc_common::errors::Level::Note | swc_common::errors::Level::Help => {
+                tracing::trace!("{}", db.message())
+            }
         }
     }
 }
 
+fn bundle_js_to_writer(
+    file: PathBuf,
+    bundle: bool,
+    minify: bool,
+    write_to: &mut impl std::io::Write,
+) -> anyhow::Result<()> {
+    let globals = Globals::new();
+    let handler = Handler::with_emitter_and_flags(Box::new(TracingEmitter), Default::default());
+    GLOBALS.set(&globals, || {
+        HANDLER.set(&handler, || {
+            bundle_js_to_writer_inside_handler(&globals, file, bundle, minify, write_to)
+        })
+    })
+}
+
+fn bundle_js_to_writer_inside_handler(
+    globals: &Globals,
+    file: PathBuf,
+    bundle: bool,
+    minify: bool,
+    write_to: &mut impl std::io::Write,
+) -> anyhow::Result<()> {
+    let cm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
+    let mut module = if bundle {
+        let node_resolver = NodeModulesResolver::new(TargetEnv::Browser, Default::default(), true);
+        let mut bundler = Bundler::new(
+            globals,
+            cm.clone(),
+            PathLoader { cm: cm.clone() },
+            node_resolver,
+            Config {
+                require: true,
+                ..Default::default()
+            },
+            Box::new(Hook),
+        );
+        let mut entries = HashMap::default();
+        entries.insert("main".to_string(), FileName::Real(file));
+
+        let mut bundles = bundler
+            .bundle(entries)
+            .context("failed to bundle javascript with swc")?;
+        // Since we only inserted one entry, there should only be one bundle in the output
+        let bundle = bundles
+            .pop()
+            .ok_or_else(|| anyhow::anyhow!("swc did not output any bundles"))?;
+        bundle.module
+    } else {
+        let fm = cm.load_file(Path::new(&file)).expect("Failed to load file");
+
+        let lexer = Lexer::new(
+            Default::default(),
+            Default::default(),
+            SourceFileInput::from(&*fm),
+            None,
+        );
+        let mut parser = Parser::new_from(lexer);
+
+        parser.parse_module().map_err(|err| {
+            HANDLER.with(|handler| {
+                let message = err.into_diagnostic(handler).message();
+                anyhow::anyhow!("{}", message)
+            })
+        })?
+    };
+
+    if minify {
+        module = swc_ecma_minifier::optimize(
+            std::mem::take(&mut module).into(),
+            cm.clone(),
+            None,
+            None,
+            &MinifyOptions {
+                rename: true,
+                compress: None,
+                mangle: None,
+                ..Default::default()
+            },
+            &ExtraOptions {
+                unresolved_mark: Mark::new(),
+                top_level_mark: Mark::new(),
+                mangle_name_cache: None,
+            },
+        )
+        .expect_module();
+        module.visit_mut_with(&mut fixer(None));
+    }
+
+    let mut emitter = swc_ecma_codegen::Emitter {
+        cfg: swc_ecma_codegen::Config::default().with_minify(minify),
+        cm: cm.clone(),
+        comments: None,
+        wr: Box::new(JsWriter::new(cm, "\n", write_to, None)),
+    };
+
+    emitter.emit_module(&module)?;
+
+    Ok(())
+}
+
+struct PathLoader {
+    cm: Lrc<SourceMap>,
+}
+
+impl Load for PathLoader {
+    fn load(&self, file: &FileName) -> anyhow::Result<ModuleData> {
+        let file = match file {
+            FileName::Real(v) => v,
+            _ => anyhow::bail!("Only real files are supported"),
+        };
+
+        let fm = self.cm.load_file(file)?;
+
+        let module = HANDLER.with(|handler| {
+            parse_file_as_module(
+                &fm,
+                Syntax::Es(Default::default()),
+                Default::default(),
+                None,
+                &mut Vec::new(),
+            )
+            .map_err(|err| {
+                let message = err.into_diagnostic(handler).message();
+                anyhow::anyhow!("{}", message)
+            })
+            .context("Failed to parse javascript")
+        })?;
+
+        Ok(ModuleData {
+            fm,
+            module,
+            helpers: Default::default(),
+        })
+    }
+}
+
+// Adapted from https://github.com/swc-project/swc/blob/624680b7896cef9d8e30bd5ff910538298016974/bindings/binding_core_node/src/bundle.rs#L266-L302
+struct Hook;
+
+impl swc_bundler::Hook for Hook {
+    fn get_import_meta_props(
+        &self,
+        span: Span,
+        module_record: &ModuleRecord,
+    ) -> Result<Vec<KeyValueProp>, Error> {
+        let file_name = module_record.file_name.to_string();
+
+        Ok(vec![
+            KeyValueProp {
+                key: PropName::Ident(IdentName::new("url".into(), span)),
+                value: Box::new(Expr::Lit(Lit::Str(Str {
+                    span,
+                    raw: None,
+                    value: file_name.into(),
+                }))),
+            },
+            KeyValueProp {
+                key: PropName::Ident(IdentName::new("main".into(), span)),
+                value: Box::new(if module_record.is_entry {
+                    Expr::Member(MemberExpr {
+                        span,
+                        obj: Box::new(Expr::MetaProp(MetaPropExpr {
+                            span,
+                            kind: MetaPropKind::ImportMeta,
+                        })),
+                        prop: MemberProp::Ident(IdentName::new("main".into(), span)),
+                    })
+                } else {
+                    Expr::Lit(Lit::Bool(Bool { span, value: false }))
+                }),
+            },
+        ])
+    }
+}
+
 pub(crate) fn process_js(
     js_options: &JsAssetOptions,
     source: &Path,
     output_path: &Path,
+    bundle: bool,
 ) -> anyhow::Result<()> {
-    let js = if js_options.minified() {
-        minify_js(source)?
+    let mut writer = std::io::BufWriter::new(std::fs::File::create(output_path)?);
+    if js_options.minified() {
+        if let Err(err) = bundle_js_to_writer(source.to_path_buf(), bundle, true, &mut writer) {
+            tracing::error!("Failed to minify js. Falling back to non-minified: {err}");
+        }
     } else {
         let mut source_file = std::fs::File::open(source)?;
-        let mut source = String::new();
-        source_file.read_to_string(&mut source)?;
-        source
-    };
-
-    std::fs::write(output_path, js).with_context(|| {
-        format!(
-            "Failed to write js to output location: {}",
-            output_path.display()
-        )
-    })?;
+        std::io::copy(&mut source_file, &mut writer).with_context(|| {
+            format!(
+                "Failed to write js to output location: {}",
+                output_path.display()
+            )
+        })?;
+    }
 
     Ok(())
 }