Restore manganis optimizations (#3195)

* Create a crate for constant serialization of config structs for manganis
* use SerializeConst for the image asset builder
* switch to a declarative macro for assets
* add serializable options for each asset type
* implement the new manganis macro
* optimize assets in the CLI
* Fix assets with dioxus formattable
* reduce fuzzing test limits
* use the new syntax in examples
* fix avif support
* Fix manganis doc tests
* cache asset optimization
* Split out asset and bundled asset
* make hash pointer width independent
* Fix manganis macro docs
* add constvec::is_empty method to fix clippy lint
* remove nasm feature
* simplify test_rsplit_once test so typos passes
* revert example change from clippy fix
* always accept unix paths
* fix windows path seperator
* fix folder asset hash
* Optimize assets in a blocking task
* Fix asset options docs
* Document Asset and BundledAsset
* simplify the linker macro a bit
* add more docs to AssetParser expansion
* add image format helpers
* Split by  both unix and windows path separators and take the smallest one

packages/cli/Cargo.toml

@@ -88,6 +88,76 @@ prettyplease = { workspace = true }
 brotli = "6.0.0"
 ignore = "0.4.22"
 env_logger = { workspace = true }
+const-serialize = { workspace = true, features = ["serde"] }
+
+# Image compression/conversion
+# - JPEG
+mozjpeg = { version = "0.10.7", default-features = false, features = [
+    "parallel",
+] }
+# - PNG
+imagequant = "4.2.0"
+png = "0.17.9"
+# Image format/conversion
+image = { version = "0.25", features = ["avif"] }
+
+# CSS Minification
+lightningcss = "1.0.0-alpha.60"
+
+# Js minification - swc has introduces minor versions with breaking changes in the past so we pin all of their crates
+swc = "=0.283.0"
+swc_allocator = { version = "=0.1.8", default-features = false }
+swc_atoms = { version = "=0.6.7", default-features = false }
+swc_cached = { version = "=0.3.20", default-features = false }
+swc_common = { version = "=0.37.5", default-features = false }
+swc_compiler_base = { version = "=0.19.0", default-features = false }
+swc_config = { version = "=0.1.15", default-features = false }
+swc_config_macro = { version = "=0.1.4", default-features = false }
+swc_ecma_ast = { version = "=0.118.2", default-features = false }
+swc_ecma_codegen = { version = "=0.155.1", default-features = false }
+swc_ecma_codegen_macros = { version = "=0.7.7", default-features = false }
+swc_ecma_compat_bugfixes = { version = "=0.12.0", default-features = false }
+swc_ecma_compat_common = { version = "=0.11.0", default-features = false }
+swc_ecma_compat_es2015 = { version = "=0.12.0", default-features = false }
+swc_ecma_compat_es2016 = { version = "=0.12.0", default-features = false }
+swc_ecma_compat_es2017 = { version = "=0.12.0", default-features = false }
+swc_ecma_compat_es2018 = { version = "=0.12.0", default-features = false }
+swc_ecma_compat_es2019 = { version = "=0.12.0", default-features = false }
+swc_ecma_compat_es2020 = { version = "=0.12.0", default-features = false }
+swc_ecma_compat_es2021 = { version = "=0.12.0", default-features = false }
+swc_ecma_compat_es2022 = { version = "=0.12.0", default-features = false }
+swc_ecma_compat_es3 = { version = "=0.12.0", default-features = false }
+swc_ecma_ext_transforms = { version = "=0.120.0", default-features = false }
+swc_ecma_lints = { version = "=0.100.0", default-features = false }
+swc_ecma_loader = { version = "=0.49.1", default-features = false }
+swc_ecma_minifier = { version = "=0.204.0", default-features = false }
+swc_ecma_parser = { version = "=0.149.1", default-features = false }
+swc_ecma_preset_env = { version = "=0.217.0", default-features = false, features = [
+    "serde",
+] }
+swc_ecma_transforms = { version = "=0.239.0", default-features = false }
+swc_ecma_transforms_base = { version = "=0.145.0", default-features = false }
+swc_ecma_transforms_classes = { version = "=0.134.0", default-features = false }
+swc_ecma_transforms_compat = { version = "=0.171.0", default-features = false }
+swc_ecma_transforms_macros = { version = "=0.5.5", default-features = false }
+swc_ecma_transforms_module = { version = "=0.190.0", default-features = false }
+swc_ecma_transforms_optimization = { version = "=0.208.0", default-features = false }
+swc_ecma_transforms_proposal = { version = "=0.178.0", default-features = false }
+swc_ecma_transforms_react = { version = "=0.191.0", default-features = false }
+swc_ecma_transforms_typescript = { version = "=0.198.1", default-features = false }
+swc_ecma_usage_analyzer = { version = "=0.30.3", default-features = false }
+swc_ecma_utils = { version = "=0.134.2", default-features = false }
+swc_ecma_visit = { version = "=0.104.8", default-features = false }
+swc_eq_ignore_macros = { version = "=0.1.4", default-features = false }
+swc_error_reporters = { version = "=0.21.0", default-features = false }
+swc_fast_graph = { version = "=0.25.0", default-features = false }
+swc_macros_common = { version = "=0.3.13", default-features = false }
+swc_node_comments = { version = "=0.24.0", default-features = false }
+swc_timer = { version = "=0.25.0", default-features = false }
+swc_trace_macro = { version = "=0.1.3", default-features = false }
+swc_transform_common = { version = "=0.1.1", default-features = false }
+swc_typescript = { version = "=0.5.0", default-features = false }
+swc_visit = { version = "=0.6.2", default-features = false }
 
 tracing-subscriber = { version = "0.3.18", features = ["std", "env-filter", "json"] }
 console-subscriber = { version = "0.3.0", optional = true }

packages/cli/src/assets/css.rs

@@ -0,0 +1,42 @@
+use std::path::Path;
+
+use anyhow::Context;
+use lightningcss::{
+    printer::PrinterOptions,
+    stylesheet::{MinifyOptions, ParserOptions, StyleSheet},
+};
+use manganis_core::CssAssetOptions;
+
+pub(crate) fn process_css(
+    css_options: &CssAssetOptions,
+    source: &Path,
+    output_path: &Path,
+) -> anyhow::Result<()> {
+    let css = std::fs::read_to_string(source)?;
+
+    let css = if css_options.minified() {
+        minify_css(&css)
+    } else {
+        css
+    };
+
+    std::fs::write(output_path, css).with_context(|| {
+        format!(
+            "Failed to write css to output location: {}",
+            output_path.display()
+        )
+    })?;
+
+    Ok(())
+}
+
+pub(crate) fn minify_css(css: &str) -> String {
+    let mut stylesheet = StyleSheet::parse(css, ParserOptions::default()).unwrap();
+    stylesheet.minify(MinifyOptions::default()).unwrap();
+    let printer = PrinterOptions {
+        minify: true,
+        ..Default::default()
+    };
+    let res = stylesheet.to_css(printer).unwrap();
+    res.code
+}

packages/cli/src/assets/file.rs

@@ -0,0 +1,75 @@
+use anyhow::Context;
+use manganis_core::{AssetOptions, CssAssetOptions, ImageAssetOptions, JsAssetOptions};
+use std::path::Path;
+
+use super::{
+    css::process_css, folder::process_folder, image::process_image, js::process_js,
+    json::process_json,
+};
+
+/// Process a specific file asset with the given options reading from the source and writing to the output path
+pub(crate) fn process_file_to(
+    options: &AssetOptions,
+    source: &Path,
+    output_path: &Path,
+) -> anyhow::Result<()> {
+    // If the file already exists, then we must have a file with the same hash
+    // already. The hash has the file contents and options, so if we find a file
+    // with the same hash, we probably already created this file in the past
+    if output_path.exists() {
+        return Ok(());
+    }
+    if let Some(parent) = output_path.parent() {
+        if !parent.exists() {
+            std::fs::create_dir_all(parent)?;
+        }
+    }
+    match options {
+        AssetOptions::Unknown => match source.extension().map(|e| e.to_string_lossy()).as_deref() {
+            Some("css") => {
+                process_css(&CssAssetOptions::new(), source, output_path)?;
+            }
+            Some("js") => {
+                process_js(&JsAssetOptions::new(), source, output_path)?;
+            }
+            Some("json") => {
+                process_json(source, output_path)?;
+            }
+            Some("jpg" | "jpeg" | "png" | "webp" | "avif") => {
+                process_image(&ImageAssetOptions::new(), source, output_path)?;
+            }
+            None if source.is_dir() => {
+                process_folder(source, output_path)?;
+            }
+            Some(_) | None => {
+                let source_file = std::fs::File::open(source)?;
+                let mut reader = std::io::BufReader::new(source_file);
+                let output_file = std::fs::File::create(output_path)?;
+                let mut writer = std::io::BufWriter::new(output_file);
+                std::io::copy(&mut reader, &mut writer).with_context(|| {
+                    format!(
+                        "Failed to write file to output location: {}",
+                        output_path.display()
+                    )
+                })?;
+            }
+        },
+        AssetOptions::Css(options) => {
+            process_css(options, source, output_path)?;
+        }
+        AssetOptions::Js(options) => {
+            process_js(options, source, output_path)?;
+        }
+        AssetOptions::Image(options) => {
+            process_image(options, source, output_path)?;
+        }
+        AssetOptions::Folder(_) => {
+            process_folder(source, output_path)?;
+        }
+        _ => {
+            tracing::warn!("Unknown asset options: {:?}", options);
+        }
+    }
+
+    Ok(())
+}

packages/cli/src/assets/folder.rs

@@ -0,0 +1,41 @@
+use std::path::Path;
+
+use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
+
+use super::file::process_file_to;
+
+/// Process a folder, optimizing and copying all assets into the output folder
+pub fn process_folder(source: &Path, output_folder: &Path) -> anyhow::Result<()> {
+    // Create the folder
+    std::fs::create_dir_all(output_folder)?;
+
+    // Then optimize children
+    let files: Vec<_> = std::fs::read_dir(source)
+        .into_iter()
+        .flatten()
+        .flatten()
+        .collect();
+
+    files.par_iter().try_for_each(|file| {
+        let file = file.path();
+        let metadata = file.metadata()?;
+        let output_path = output_folder.join(file.strip_prefix(source)?);
+        if metadata.is_dir() {
+            process_folder(&file, &output_path)
+        } else {
+            process_file_minimal(&file, &output_path)
+        }
+    })?;
+
+    Ok(())
+}
+
+/// Optimize a file without changing any of its contents significantly (e.g. by changing the extension)
+fn process_file_minimal(input_path: &Path, output_path: &Path) -> anyhow::Result<()> {
+    process_file_to(
+        &manganis_core::AssetOptions::Unknown,
+        input_path,
+        output_path,
+    )?;
+    Ok(())
+}

packages/cli/src/assets/image/jpg.rs

@@ -0,0 +1,23 @@
+use image::{DynamicImage, EncodableLayout};
+use std::{
+    io::{BufWriter, Write},
+    path::Path,
+};
+
+pub(crate) fn compress_jpg(image: DynamicImage, output_location: &Path) -> anyhow::Result<()> {
+    let mut comp = mozjpeg::Compress::new(mozjpeg::ColorSpace::JCS_EXT_RGBX);
+    let width = image.width() as usize;
+    let height = image.height() as usize;
+
+    comp.set_size(width, height);
+    let mut comp = comp.start_compress(Vec::new())?; // any io::Write will work
+
+    comp.write_scanlines(image.to_rgba8().as_bytes())?;
+
+    let jpeg_bytes = comp.finish()?;
+
+    let file = std::fs::File::create(output_location)?;
+    let w = &mut BufWriter::new(file);
+    w.write_all(&jpeg_bytes)?;
+    Ok(())
+}

packages/cli/src/assets/image/mod.rs

@@ -0,0 +1,62 @@
+use std::path::Path;
+
+use anyhow::Context;
+use jpg::compress_jpg;
+use manganis_core::{ImageAssetOptions, ImageFormat, ImageSize};
+use png::compress_png;
+
+mod jpg;
+mod png;
+
+pub(crate) fn process_image(
+    image_options: &ImageAssetOptions,
+    source: &Path,
+    output_path: &Path,
+) -> anyhow::Result<()> {
+    let mut image = image::ImageReader::new(std::io::Cursor::new(&*std::fs::read(source)?))
+        .with_guessed_format()?
+        .decode();
+
+    if let Ok(image) = &mut image {
+        if let ImageSize::Manual { width, height } = image_options.size() {
+            *image = image.resize_exact(width, height, image::imageops::FilterType::Lanczos3);
+        }
+    }
+
+    match (image, image_options.format()) {
+        (image, ImageFormat::Png) => {
+            compress_png(image?, output_path);
+        }
+        (image, ImageFormat::Jpg) => {
+            compress_jpg(image?, output_path)?;
+        }
+        (Ok(image), ImageFormat::Avif) => {
+            if let Err(error) = image.save(output_path) {
+                tracing::error!("Failed to save avif image: {} with path {}. You must have the avif feature enabled to use avif assets", error, output_path.display());
+            }
+        }
+        (Ok(image), ImageFormat::Webp) => {
+            if let Err(err) = image.save(output_path) {
+                tracing::error!("Failed to save webp image: {}. You must have the avif feature enabled to use webp assets", err);
+            }
+        }
+        (Ok(image), _) => {
+            image.save(output_path)?;
+        }
+        // If we can't decode the image or it is of an unknown type, we just copy the file
+        _ => {
+            let source_file = std::fs::File::open(source)?;
+            let mut reader = std::io::BufReader::new(source_file);
+            let output_file = std::fs::File::create(output_path)?;
+            let mut writer = std::io::BufWriter::new(output_file);
+            std::io::copy(&mut reader, &mut writer).with_context(|| {
+                format!(
+                    "Failed to write image to output location: {}",
+                    output_path.display()
+                )
+            })?;
+        }
+    }
+
+    Ok(())
+}

packages/cli/src/assets/image/png.rs

@@ -0,0 +1,52 @@
+use std::{io::BufWriter, path::Path};
+
+use image::DynamicImage;
+
+pub(crate) fn compress_png(image: DynamicImage, output_location: &Path) {
+    // Image loading/saving is outside scope of this library
+    let width = image.width() as usize;
+    let height = image.height() as usize;
+    let bitmap: Vec<_> = image
+        .into_rgba8()
+        .pixels()
+        .map(|px| imagequant::RGBA::new(px[0], px[1], px[2], px[3]))
+        .collect();
+
+    // Configure the library
+    let mut liq = imagequant::new();
+    liq.set_speed(5).unwrap();
+    liq.set_quality(0, 99).unwrap();
+
+    // Describe the bitmap
+    let mut img = liq.new_image(&bitmap[..], width, height, 0.0).unwrap();
+
+    // The magic happens in quantize()
+    let mut res = match liq.quantize(&mut img) {
+        Ok(res) => res,
+        Err(err) => panic!("Quantization failed, because: {err:?}"),
+    };
+
+    let (palette, pixels) = res.remapped(&mut img).unwrap();
+
+    let file = std::fs::File::create(output_location).unwrap();
+    let w = &mut BufWriter::new(file);
+
+    let mut encoder = png::Encoder::new(w, width as u32, height as u32);
+    encoder.set_color(png::ColorType::Rgba);
+    let mut flattened_palette = Vec::new();
+    let mut alpha_palette = Vec::new();
+    for px in palette {
+        flattened_palette.push(px.r);
+        flattened_palette.push(px.g);
+        flattened_palette.push(px.b);
+        alpha_palette.push(px.a);
+    }
+    encoder.set_palette(flattened_palette);
+    encoder.set_trns(alpha_palette);
+    encoder.set_depth(png::BitDepth::Eight);
+    encoder.set_color(png::ColorType::Indexed);
+    encoder.set_compression(png::Compression::Best);
+    let mut writer = encoder.write_header().unwrap();
+    writer.write_image_data(&pixels).unwrap();
+    writer.finish().unwrap();
+}

packages/cli/src/assets/js.rs

@@ -0,0 +1,69 @@
+use std::io::Read;
+use std::path::Path;
+use std::sync::Arc;
+
+use anyhow::Context;
+use manganis_core::JsAssetOptions;
+use swc::{config::JsMinifyOptions, try_with_handler, BoolOrDataConfig};
+use swc_common::{sync::Lrc, FileName};
+use swc_common::{SourceMap, GLOBALS};
+
+pub(crate) fn minify_js(source: &Path) -> anyhow::Result<String> {
+    let mut source_file = std::fs::File::open(source)?;
+    let cm = Arc::new(SourceMap::default());
+
+    let mut js = String::new();
+    source_file.read_to_string(&mut js)?;
+    let c = swc::Compiler::new(cm.clone());
+    let output = GLOBALS
+        .set(&Default::default(), || {
+            try_with_handler(cm.clone(), Default::default(), |handler| {
+                let filename = Lrc::new(FileName::Real(source.to_path_buf()));
+                let fm = cm.new_source_file(filename, js.to_string());
+
+                c.minify(
+                    fm,
+                    handler,
+                    &JsMinifyOptions {
+                        compress: BoolOrDataConfig::from_bool(true),
+                        mangle: BoolOrDataConfig::from_bool(true),
+                        ..Default::default()
+                    },
+                )
+                .context("failed to minify javascript")
+            })
+        })
+        .map(|output| output.code);
+
+    match output {
+        Ok(output) => Ok(output),
+        Err(err) => {
+            tracing::error!("Failed to minify javascript: {}", err);
+            Ok(js)
+        }
+    }
+}
+
+pub(crate) fn process_js(
+    js_options: &JsAssetOptions,
+    source: &Path,
+    output_path: &Path,
+) -> anyhow::Result<()> {
+    let js = if js_options.minified() {
+        minify_js(source)?
+    } else {
+        let mut source_file = std::fs::File::open(source)?;
+        let mut source = String::new();
+        source_file.read_to_string(&mut source)?;
+        source
+    };
+
+    std::fs::write(output_path, js).with_context(|| {
+        format!(
+            "Failed to write js to output location: {}",
+            output_path.display()
+        )
+    })?;
+
+    Ok(())
+}

packages/cli/src/assets/json.rs

@@ -0,0 +1,33 @@
+use std::{io::Read, path::Path};
+
+use anyhow::Context;
+
+pub(crate) fn minify_json(source: &str) -> anyhow::Result<String> {
+    // First try to parse the json
+    let json: serde_json::Value = serde_json::from_str(source)?;
+    // Then print it in a minified format
+    let json = serde_json::to_string(&json)?;
+    Ok(json)
+}
+
+pub(crate) fn process_json(source: &Path, output_path: &Path) -> anyhow::Result<()> {
+    let mut source_file = std::fs::File::open(source)?;
+    let mut source = String::new();
+    source_file.read_to_string(&mut source)?;
+    let json = match minify_json(&source) {
+        Ok(json) => json,
+        Err(err) => {
+            tracing::error!("Failed to minify json: {}", err);
+            source
+        }
+    };
+
+    std::fs::write(output_path, json).with_context(|| {
+        format!(
+            "Failed to write json to output location: {}",
+            output_path.display()
+        )
+    })?;
+
+    Ok(())
+}

packages/cli/src/assets/mod.rs

@@ -1,24 +1,34 @@
 use anyhow::Context;
-use manganis_core::{LinkSection, ResourceAsset};
+use manganis_core::linker::LinkSection;
+use manganis_core::BundledAsset;
 use object::{read::archive::ArchiveFile, File as ObjectFile, Object, ObjectSection};
 use serde::{Deserialize, Serialize};
 use std::path::Path;
 use std::{collections::HashMap, path::PathBuf};
 
+mod css;
+mod file;
+mod folder;
+mod image;
+mod js;
+mod json;
+
+pub(crate) use file::process_file_to;
+
 /// A manifest of all assets collected from dependencies
 ///
 /// This will be filled in primarily by incremental compilation artifacts.
 #[derive(Debug, PartialEq, Default, Clone, Serialize, Deserialize)]
 pub(crate) struct AssetManifest {
     /// Map of bundled asset name to the asset itself
-    pub(crate) assets: HashMap<PathBuf, ResourceAsset>,
+    pub(crate) assets: HashMap<PathBuf, BundledAsset>,
 }
 
 impl AssetManifest {
     #[allow(dead_code)]
     pub(crate) fn load_from_file(path: &Path) -> anyhow::Result<Self> {
-        let src = std::fs::read_to_string(path)
-            .context("Failed to read asset manifest from filesystem")?;
+        let src = std::fs::read_to_string(path)?;
+
         serde_json::from_str(&src)
             .with_context(|| format!("Failed to parse asset manifest from {path:?}\n{src}"))
     }
@@ -44,7 +54,7 @@ impl AssetManifest {
         Ok(())
     }
 
-    /// Fill this manifest from an rlib / ar file that contains many object files and their entryies
+    /// Fill this manifest from an rlib / ar file that contains many object files and their entries
     fn add_from_archive_file(&mut self, archive: &ArchiveFile, data: &[u8]) -> object::Result<()> {
         // Look through each archive member for object files.
         // Read the archive member's binary data (we know it's an object file)
@@ -84,17 +94,13 @@ impl AssetManifest {
                 .uncompressed_data()
                 .context("Could not read uncompressed data from object file")?;
 
-            let as_str = std::str::from_utf8(&bytes)
-                .context("object file contained non utf8 encoding")?
-                .chars()
-                .filter(|c| !c.is_control())
-                .collect::<String>();
-
-            let assets = serde_json::Deserializer::from_str(&as_str).into_iter::<ResourceAsset>();
-            for as_resource in assets.flatten() {
-                // Some platforms (e.g. macOS) start the manganis section with a null byte, we need to filter that out before we deserialize the JSON
+            let mut buffer = const_serialize::ConstReadBuffer::new(&bytes);
+            while let Some((remaining_buffer, asset)) =
+                const_serialize::deserialize_const!(BundledAsset, buffer)
+            {
                 self.assets
-                    .insert(as_resource.absolute.clone(), as_resource);
+                    .insert(asset.absolute_source_path().into(), asset);
+                buffer = remaining_buffer;
             }
         }
 

packages/cli/src/build/bundle.rs

@@ -1,12 +1,14 @@
+use crate::assets::process_file_to;
 use crate::Result;
 use crate::{assets::AssetManifest, TraceSrc};
 use crate::{BuildRequest, Platform};
 use anyhow::Context;
+use manganis_core::AssetOptions;
 use rayon::prelude::{IntoParallelRefIterator, ParallelIterator};
-use std::{
-    fs::create_dir_all,
-    path::{Path, PathBuf},
-};
+use std::collections::HashSet;
+use std::future::Future;
+use std::path::{Path, PathBuf};
+use std::pin::Pin;
 use std::{sync::atomic::AtomicUsize, time::Duration};
 use tokio::process::Command;
 use wasm_bindgen_cli_support::Bindgen;
@@ -377,21 +379,54 @@ impl AppBundle {
 
         let asset_dir = self.build.asset_dir();
 
-        // First, clear the asset dir
-        // todo(jon): cache the asset dir, removing old files and only copying new ones that changed since the last build
-        _ = std::fs::remove_dir_all(&asset_dir);
-        _ = create_dir_all(&asset_dir);
+        // First, clear the asset dir of any files that don't exist in the new manifest
+        _ = tokio::fs::create_dir_all(&asset_dir).await;
+        // Create a set of all the paths that new files will be bundled to
+        let bundled_output_paths: HashSet<_> = self
+            .app
+            .assets
+            .assets
+            .values()
+            .map(|a| asset_dir.join(a.bundled_path()))
+            .collect();
+        // one possible implementation of walking a directory only visiting files
+        fn remove_old_assets<'a>(
+            path: &'a Path,
+            bundled_output_paths: &'a HashSet<PathBuf>,
+        ) -> Pin<Box<dyn Future<Output = std::io::Result<()>> + Send + 'a>> {
+            Box::pin(async move {
+                // If this asset is in the manifest, we don't need to remove it
+                if bundled_output_paths.contains(path.canonicalize()?.as_path()) {
+                    return Ok(());
+                }
+                // Otherwise, if it is a directory, we need to walk it and remove child files
+                if path.is_dir() {
+                    for entry in std::fs::read_dir(path)?.flatten() {
+                        let path = entry.path();
+                        remove_old_assets(&path, bundled_output_paths).await?;
+                    }
+                    if path.read_dir()?.next().is_none() {
+                        // If the directory is empty, remove it
+                        tokio::fs::remove_dir(path).await?;
+                    }
+                } else {
+                    // If it is a file, remove it
+                    tokio::fs::remove_file(path).await?;
+                }
+                Ok(())
+            })
+        }
+        remove_old_assets(&asset_dir, &bundled_output_paths).await?;
 
         // todo(jon): we also want to eventually include options for each asset's optimization and compression, which we currently aren't
         let mut assets_to_transfer = vec![];
 
         // Queue the bundled assets
-        for asset in self.app.assets.assets.keys() {
-            let bundled = self.app.assets.assets.get(asset).unwrap();
-            let from = bundled.absolute.clone();
-            let to = asset_dir.join(&bundled.bundled);
+        for (asset, bundled) in &self.app.assets.assets {
+            let from = asset.clone();
+            let to = asset_dir.join(bundled.bundled_path());
             tracing::debug!("Copying asset {from:?} to {to:?}");
-            assets_to_transfer.push((from, to));
+            assets_to_transfer.push((from, to, *bundled.options()));
         }
 
         // And then queue the legacy assets
@@ -399,35 +434,42 @@ impl AppBundle {
         for from in self.build.krate.legacy_asset_dir_files() {
             let to = asset_dir.join(from.file_name().unwrap());
             tracing::debug!("Copying legacy asset {from:?} to {to:?}");
-            assets_to_transfer.push((from, to));
+            assets_to_transfer.push((from, to, AssetOptions::Unknown));
         }
 
         let asset_count = assets_to_transfer.len();
         let assets_finished = AtomicUsize::new(0);
 
         // Parallel Copy over the assets and keep track of progress with an atomic counter
-        // todo: we want to use the fastfs variant that knows how to parallelize folders, too
-        assets_to_transfer.par_iter().try_for_each(|(from, to)| {
-            tracing::trace!(
-                "Starting asset copy {current}/{asset_count} from {from:?}",
-                current = assets_finished.fetch_add(0, std::sync::atomic::Ordering::SeqCst),
-            );
+        let progress = self.build.progress.clone();
+        // Optimizing assets is expensive and blocking, so we do it in a tokio spawn blocking task
+        tokio::task::spawn_blocking(move || {
+            assets_to_transfer
+                .par_iter()
+                .try_for_each(|(from, to, options)| {
+                    tracing::trace!(
+                        "Starting asset copy {current}/{asset_count} from {from:?}",
+                        current = assets_finished.fetch_add(0, std::sync::atomic::Ordering::SeqCst),
+                    );
 
-            // todo(jon): implement optimize + pre_compress on the asset type
-            let res = crate::fastfs::copy_asset(from, to);
+                    let res = process_file_to(options, from, to);
 
-            if let Err(err) = res.as_ref() {
-                tracing::error!("Failed to copy asset {from:?}: {err}");
-            }
+                    if let Err(err) = res.as_ref() {
+                        tracing::error!("Failed to copy asset {from:?}: {err}");
+                    }
 
-            self.build.status_copied_asset(
-                assets_finished.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + 1,
-                asset_count,
-                from.clone(),
-            );
+                    BuildRequest::status_copied_asset(
+                        &progress,
+                        assets_finished.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + 1,
+                        asset_count,
+                        from.to_path_buf(),
+                    );
 
-            res.map(|_| ())
-        })?;
+                    res.map(|_| ())
+                })
+        })
+        .await
+        .map_err(|e| anyhow::anyhow!("A task failed while trying to copy assets: {e}"))??;
 
         Ok(())
     }

packages/cli/src/build/progress.rs

@@ -59,8 +59,13 @@ impl BuildRequest {
         });
     }
 
-    pub(crate) fn status_copied_asset(&self, current: usize, total: usize, path: PathBuf) {
-        _ = self.progress.unbounded_send(BuildUpdate::Progress {
+    pub(crate) fn status_copied_asset(
+        progress: &UnboundedSender<BuildUpdate>,
+        current: usize,
+        total: usize,
+        path: PathBuf,
+    ) {
+        _ = progress.unbounded_send(BuildUpdate::Progress {
             stage: BuildStage::CopyingAssets {
                 current,
                 total,

packages/cli/src/build/request.rs

@@ -558,7 +558,7 @@ impl BuildRequest {
         static INITIALIZED: OnceCell<Result<()>> = OnceCell::new();
 
         let success = INITIALIZED.get_or_init(|| {
-            _ = remove_dir_all(self.root_dir());
+            _ = remove_dir_all(self.exe_dir());
 
             create_dir_all(self.root_dir())?;
             create_dir_all(self.exe_dir())?;

packages/cli/src/serve/handle.rs

@@ -181,8 +181,8 @@ impl AppHandle {
 
         // The asset might've been renamed thanks to the manifest, let's attempt to reload that too
         if let Some(resource) = self.app.app.assets.assets.get(changed_file).as_ref() {
-            let res = std::fs::copy(changed_file, asset_dir.join(&resource.bundled));
-            bundled_name = Some(PathBuf::from(&resource.bundled));
+            let res = std::fs::copy(changed_file, asset_dir.join(resource.bundled_path()));
+            bundled_name = Some(PathBuf::from(resource.bundled_path()));
             if let Err(e) = res {
                 tracing::debug!("Failed to hotreload asset {e}");
             }