improve build times by extracting asset opt into its own crate (#3273)

* improve build types by extracing asset opt as a dylib
* dont use dylib, but let regular workspace caching do the heavy lifting
* add prebuild to cli for better lock contention
* hoist deps
* disable slow tests
* re-enable sequential, debug=0 for dev
* use vfs for windows
* dont cache on failure, fix windows vfs caceh
* free disk space re-order
* add clippy + fmt to windows channel
* yoink makefile
* proper rustfmt/clippy on toolchain
* set incremental to zero since cargo-cache disables it

packages/cli-opt/src/css.rs

@@ -0,0 +1,42 @@
+use std::path::Path;
+
+use anyhow::Context;
+use lightningcss::{
+    printer::PrinterOptions,
+    stylesheet::{MinifyOptions, ParserOptions, StyleSheet},
+};
+use manganis_core::CssAssetOptions;
+
+pub(crate) fn process_css(
+    css_options: &CssAssetOptions,
+    source: &Path,
+    output_path: &Path,
+) -> anyhow::Result<()> {
+    let css = std::fs::read_to_string(source)?;
+
+    let css = if css_options.minified() {
+        minify_css(&css)
+    } else {
+        css
+    };
+
+    std::fs::write(output_path, css).with_context(|| {
+        format!(
+            "Failed to write css to output location: {}",
+            output_path.display()
+        )
+    })?;
+
+    Ok(())
+}
+
+pub(crate) fn minify_css(css: &str) -> String {
+    let mut stylesheet = StyleSheet::parse(css, ParserOptions::default()).unwrap();
+    stylesheet.minify(MinifyOptions::default()).unwrap();
+    let printer = PrinterOptions {
+        minify: true,
+        ..Default::default()
+    };
+    let res = stylesheet.to_css(printer).unwrap();
+    res.code
+}

packages/cli-opt/src/file.rs

@@ -0,0 +1,76 @@
+use anyhow::Context;
+use manganis_core::{AssetOptions, CssAssetOptions, ImageAssetOptions, JsAssetOptions};
+use std::path::Path;
+
+use super::{
+    css::process_css, folder::process_folder, image::process_image, js::process_js,
+    json::process_json,
+};
+
+/// Process a specific file asset with the given options reading from the source and writing to the output path
+pub fn process_file_to(
+    options: &AssetOptions,
+    source: &Path,
+    output_path: &Path,
+) -> anyhow::Result<()> {
+    // If the file already exists, then we must have a file with the same hash
+    // already. The hash has the file contents and options, so if we find a file
+    // with the same hash, we probably already created this file in the past
+    if output_path.exists() {
+        return Ok(());
+    }
+    if let Some(parent) = output_path.parent() {
+        if !parent.exists() {
+            std::fs::create_dir_all(parent)?;
+        }
+    }
+    match options {
+        AssetOptions::Unknown => match source.extension().map(|e| e.to_string_lossy()).as_deref() {
+            Some("css") => {
+                process_css(&CssAssetOptions::new(), source, output_path)?;
+            }
+            Some("js") => {
+                process_js(&JsAssetOptions::new(), source, output_path)?;
+            }
+            Some("json") => {
+                process_json(source, output_path)?;
+            }
+            Some("jpg" | "jpeg" | "png" | "webp" | "avif") => {
+                process_image(&ImageAssetOptions::new(), source, output_path)?;
+            }
+            Some(_) | None => {
+                if source.is_dir() {
+                    process_folder(source, output_path)?;
+                } else {
+                    let source_file = std::fs::File::open(source)?;
+                    let mut reader = std::io::BufReader::new(source_file);
+                    let output_file = std::fs::File::create(output_path)?;
+                    let mut writer = std::io::BufWriter::new(output_file);
+                    std::io::copy(&mut reader, &mut writer).with_context(|| {
+                        format!(
+                            "Failed to write file to output location: {}",
+                            output_path.display()
+                        )
+                    })?;
+                }
+            }
+        },
+        AssetOptions::Css(options) => {
+            process_css(options, source, output_path)?;
+        }
+        AssetOptions::Js(options) => {
+            process_js(options, source, output_path)?;
+        }
+        AssetOptions::Image(options) => {
+            process_image(options, source, output_path)?;
+        }
+        AssetOptions::Folder(_) => {
+            process_folder(source, output_path)?;
+        }
+        _ => {
+            tracing::warn!("Unknown asset options: {:?}", options);
+        }
+    }
+
+    Ok(())
+}

packages/cli-opt/src/folder.rs

@@ -0,0 +1,41 @@
+use std::path::Path;
+
+use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
+
+use super::file::process_file_to;
+
+/// Process a folder, optimizing and copying all assets into the output folder
+pub fn process_folder(source: &Path, output_folder: &Path) -> anyhow::Result<()> {
+    // Create the folder
+    std::fs::create_dir_all(output_folder)?;
+
+    // Then optimize children
+    let files: Vec<_> = std::fs::read_dir(source)
+        .into_iter()
+        .flatten()
+        .flatten()
+        .collect();
+
+    files.par_iter().try_for_each(|file| {
+        let file = file.path();
+        let metadata = file.metadata()?;
+        let output_path = output_folder.join(file.strip_prefix(source)?);
+        if metadata.is_dir() {
+            process_folder(&file, &output_path)
+        } else {
+            process_file_minimal(&file, &output_path)
+        }
+    })?;
+
+    Ok(())
+}
+
+/// Optimize a file without changing any of its contents significantly (e.g. by changing the extension)
+fn process_file_minimal(input_path: &Path, output_path: &Path) -> anyhow::Result<()> {
+    process_file_to(
+        &manganis_core::AssetOptions::Unknown,
+        input_path,
+        output_path,
+    )?;
+    Ok(())
+}

packages/cli-opt/src/image/jpg.rs

@@ -0,0 +1,23 @@
+use image::{DynamicImage, EncodableLayout};
+use std::{
+    io::{BufWriter, Write},
+    path::Path,
+};
+
+pub(crate) fn compress_jpg(image: DynamicImage, output_location: &Path) -> anyhow::Result<()> {
+    let mut comp = mozjpeg::Compress::new(mozjpeg::ColorSpace::JCS_EXT_RGBX);
+    let width = image.width() as usize;
+    let height = image.height() as usize;
+
+    comp.set_size(width, height);
+    let mut comp = comp.start_compress(Vec::new())?; // any io::Write will work
+
+    comp.write_scanlines(image.to_rgba8().as_bytes())?;
+
+    let jpeg_bytes = comp.finish()?;
+
+    let file = std::fs::File::create(output_location)?;
+    let w = &mut BufWriter::new(file);
+    w.write_all(&jpeg_bytes)?;
+    Ok(())
+}

packages/cli-opt/src/image/mod.rs

@@ -0,0 +1,62 @@
+use std::path::Path;
+
+use anyhow::Context;
+use jpg::compress_jpg;
+use manganis_core::{ImageAssetOptions, ImageFormat, ImageSize};
+use png::compress_png;
+
+mod jpg;
+mod png;
+
+pub(crate) fn process_image(
+    image_options: &ImageAssetOptions,
+    source: &Path,
+    output_path: &Path,
+) -> anyhow::Result<()> {
+    let mut image = image::ImageReader::new(std::io::Cursor::new(&*std::fs::read(source)?))
+        .with_guessed_format()?
+        .decode();
+
+    if let Ok(image) = &mut image {
+        if let ImageSize::Manual { width, height } = image_options.size() {
+            *image = image.resize_exact(width, height, image::imageops::FilterType::Lanczos3);
+        }
+    }
+
+    match (image, image_options.format()) {
+        (image, ImageFormat::Png) => {
+            compress_png(image?, output_path);
+        }
+        (image, ImageFormat::Jpg) => {
+            compress_jpg(image?, output_path)?;
+        }
+        (Ok(image), ImageFormat::Avif) => {
+            if let Err(error) = image.save(output_path) {
+                tracing::error!("Failed to save avif image: {} with path {}. You must have the avif feature enabled to use avif assets", error, output_path.display());
+            }
+        }
+        (Ok(image), ImageFormat::Webp) => {
+            if let Err(err) = image.save(output_path) {
+                tracing::error!("Failed to save webp image: {}. You must have the avif feature enabled to use webp assets", err);
+            }
+        }
+        (Ok(image), _) => {
+            image.save(output_path)?;
+        }
+        // If we can't decode the image or it is of an unknown type, we just copy the file
+        _ => {
+            let source_file = std::fs::File::open(source)?;
+            let mut reader = std::io::BufReader::new(source_file);
+            let output_file = std::fs::File::create(output_path)?;
+            let mut writer = std::io::BufWriter::new(output_file);
+            std::io::copy(&mut reader, &mut writer).with_context(|| {
+                format!(
+                    "Failed to write image to output location: {}",
+                    output_path.display()
+                )
+            })?;
+        }
+    }
+
+    Ok(())
+}

packages/cli-opt/src/image/png.rs

@@ -0,0 +1,52 @@
+use std::{io::BufWriter, path::Path};
+
+use image::DynamicImage;
+
+pub(crate) fn compress_png(image: DynamicImage, output_location: &Path) {
+    // Image loading/saving is outside scope of this library
+    let width = image.width() as usize;
+    let height = image.height() as usize;
+    let bitmap: Vec<_> = image
+        .into_rgba8()
+        .pixels()
+        .map(|px| imagequant::RGBA::new(px[0], px[1], px[2], px[3]))
+        .collect();
+
+    // Configure the library
+    let mut liq = imagequant::new();
+    liq.set_speed(5).unwrap();
+    liq.set_quality(0, 99).unwrap();
+
+    // Describe the bitmap
+    let mut img = liq.new_image(&bitmap[..], width, height, 0.0).unwrap();
+
+    // The magic happens in quantize()
+    let mut res = match liq.quantize(&mut img) {
+        Ok(res) => res,
+        Err(err) => panic!("Quantization failed, because: {err:?}"),
+    };
+
+    let (palette, pixels) = res.remapped(&mut img).unwrap();
+
+    let file = std::fs::File::create(output_location).unwrap();
+    let w = &mut BufWriter::new(file);
+
+    let mut encoder = png::Encoder::new(w, width as u32, height as u32);
+    encoder.set_color(png::ColorType::Rgba);
+    let mut flattened_palette = Vec::new();
+    let mut alpha_palette = Vec::new();
+    for px in palette {
+        flattened_palette.push(px.r);
+        flattened_palette.push(px.g);
+        flattened_palette.push(px.b);
+        alpha_palette.push(px.a);
+    }
+    encoder.set_palette(flattened_palette);
+    encoder.set_trns(alpha_palette);
+    encoder.set_depth(png::BitDepth::Eight);
+    encoder.set_color(png::ColorType::Indexed);
+    encoder.set_compression(png::Compression::Best);
+    let mut writer = encoder.write_header().unwrap();
+    writer.write_image_data(&pixels).unwrap();
+    writer.finish().unwrap();
+}

packages/cli-opt/src/js.rs

@@ -0,0 +1,69 @@
+use std::io::Read;
+use std::path::Path;
+use std::sync::Arc;
+
+use anyhow::Context;
+use manganis_core::JsAssetOptions;
+use swc::{config::JsMinifyOptions, try_with_handler, BoolOrDataConfig};
+use swc_common::{sync::Lrc, FileName};
+use swc_common::{SourceMap, GLOBALS};
+
+pub(crate) fn minify_js(source: &Path) -> anyhow::Result<String> {
+    let mut source_file = std::fs::File::open(source)?;
+    let cm = Arc::new(SourceMap::default());
+
+    let mut js = String::new();
+    source_file.read_to_string(&mut js)?;
+    let c = swc::Compiler::new(cm.clone());
+    let output = GLOBALS
+        .set(&Default::default(), || {
+            try_with_handler(cm.clone(), Default::default(), |handler| {
+                let filename = Lrc::new(FileName::Real(source.to_path_buf()));
+                let fm = cm.new_source_file(filename, js.to_string());
+
+                c.minify(
+                    fm,
+                    handler,
+                    &JsMinifyOptions {
+                        compress: BoolOrDataConfig::from_bool(true),
+                        mangle: BoolOrDataConfig::from_bool(true),
+                        ..Default::default()
+                    },
+                )
+                .context("failed to minify javascript")
+            })
+        })
+        .map(|output| output.code);
+
+    match output {
+        Ok(output) => Ok(output),
+        Err(err) => {
+            tracing::error!("Failed to minify javascript: {}", err);
+            Ok(js)
+        }
+    }
+}
+
+pub(crate) fn process_js(
+    js_options: &JsAssetOptions,
+    source: &Path,
+    output_path: &Path,
+) -> anyhow::Result<()> {
+    let js = if js_options.minified() {
+        minify_js(source)?
+    } else {
+        let mut source_file = std::fs::File::open(source)?;
+        let mut source = String::new();
+        source_file.read_to_string(&mut source)?;
+        source
+    };
+
+    std::fs::write(output_path, js).with_context(|| {
+        format!(
+            "Failed to write js to output location: {}",
+            output_path.display()
+        )
+    })?;
+
+    Ok(())
+}

packages/cli-opt/src/json.rs

@@ -0,0 +1,33 @@
+use std::{io::Read, path::Path};
+
+use anyhow::Context;
+
+pub(crate) fn minify_json(source: &str) -> anyhow::Result<String> {
+    // First try to parse the json
+    let json: serde_json::Value = serde_json::from_str(source)?;
+    // Then print it in a minified format
+    let json = serde_json::to_string(&json)?;
+    Ok(json)
+}
+
+pub(crate) fn process_json(source: &Path, output_path: &Path) -> anyhow::Result<()> {
+    let mut source_file = std::fs::File::open(source)?;
+    let mut source = String::new();
+    source_file.read_to_string(&mut source)?;
+    let json = match minify_json(&source) {
+        Ok(json) => json,
+        Err(err) => {
+            tracing::error!("Failed to minify json: {}", err);
+            source
+        }
+    };
+
+    std::fs::write(output_path, json).with_context(|| {
+        format!(
+            "Failed to write json to output location: {}",
+            output_path.display()
+        )
+    })?;
+
+    Ok(())
+}

packages/cli-opt/src/lib.rs

@@ -0,0 +1,109 @@
+use anyhow::Context;
+use manganis_core::linker::LinkSection;
+use manganis_core::BundledAsset;
+use object::{read::archive::ArchiveFile, File as ObjectFile, Object, ObjectSection};
+use serde::{Deserialize, Serialize};
+use std::path::Path;
+use std::{collections::HashMap, path::PathBuf};
+
+mod css;
+mod file;
+mod folder;
+mod image;
+mod js;
+mod json;
+
+pub use file::process_file_to;
+
+/// A manifest of all assets collected from dependencies
+///
+/// This will be filled in primarily by incremental compilation artifacts.
+#[derive(Debug, PartialEq, Default, Clone, Serialize, Deserialize)]
+pub struct AssetManifest {
+    /// Map of bundled asset name to the asset itself
+    pub assets: HashMap<PathBuf, BundledAsset>,
+}
+
+impl AssetManifest {
+    #[allow(dead_code)]
+    pub fn load_from_file(path: &Path) -> anyhow::Result<Self> {
+        let src = std::fs::read_to_string(path)?;
+
+        serde_json::from_str(&src)
+            .with_context(|| format!("Failed to parse asset manifest from {path:?}\n{src}"))
+    }
+
+    /// Fill this manifest with a file object/rlib files, typically extracted from the linker intercepted
+    pub fn add_from_object_path(&mut self, path: &Path) -> anyhow::Result<()> {
+        let data = std::fs::read(path)?;
+
+        match path.extension().and_then(|ext| ext.to_str()) {
+            // Parse an rlib as a collection of objects
+            Some("rlib") => {
+                if let Ok(archive) = object::read::archive::ArchiveFile::parse(&*data) {
+                    self.add_from_archive_file(&archive, &data)?;
+                }
+            }
+            _ => {
+                if let Ok(object) = object::File::parse(&*data) {
+                    self.add_from_object_file(&object)?;
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Fill this manifest from an rlib / ar file that contains many object files and their entries
+    fn add_from_archive_file(&mut self, archive: &ArchiveFile, data: &[u8]) -> object::Result<()> {
+        // Look through each archive member for object files.
+        // Read the archive member's binary data (we know it's an object file)
+        // And parse it with the normal `object::File::parse` to find the manganis string.
+        for member in archive.members() {
+            let member = member?;
+            let name = String::from_utf8_lossy(member.name()).to_string();
+
+            // Check if the archive member is an object file and parse it.
+            if name.ends_with(".o") {
+                let data = member.data(data)?;
+                let object = object::File::parse(data)?;
+                _ = self.add_from_object_file(&object);
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Fill this manifest with whatever tables might come from the object file
+    fn add_from_object_file(&mut self, obj: &ObjectFile) -> anyhow::Result<()> {
+        for section in obj.sections() {
+            let Ok(section_name) = section.name() else {
+                continue;
+            };
+
+            // Check if the link section matches the asset section for one of the platforms we support. This may not be the current platform if the user is cross compiling
+            let matches = LinkSection::ALL
+                .iter()
+                .any(|x| x.link_section == section_name);
+
+            if !matches {
+                continue;
+            }
+
+            let bytes = section
+                .uncompressed_data()
+                .context("Could not read uncompressed data from object file")?;
+
+            let mut buffer = const_serialize::ConstReadBuffer::new(&bytes);
+            while let Some((remaining_buffer, asset)) =
+                const_serialize::deserialize_const!(BundledAsset, buffer)
+            {
+                self.assets
+                    .insert(asset.absolute_source_path().into(), asset);
+                buffer = remaining_buffer;
+            }
+        }
+
+        Ok(())
+    }
+}