Move asset hashing into the CLI (#3988)

* Fix deduping assets

* hash assets in the linker

* fix extracting assets

* Fix the hashed asset name

* deserialize the asset back from the link section

* prevent the read from being inlined

* remove some extra logs

* Fix liveview routing

* Only look for direct dependencies on dioxus features

* fix wasm split test

* On wasm, create a new object file with the asset information

* Fix multiple assets

* add a regression test for 3467

* fix image link

* fix linker out file

* fix linux linker args

* add windows linker command

* fix linux link args

* fix windows target triple

* Expose an option to build assets in the linker intercept

* Fix the standalone optimizer and document it in the dx help command

* include the cli opt version in the hash

* fix clippy

* fix ssg port

* add a safety note to the read_volatile call

* move found assets log

* fix some merge conflicts

* fix build

* make most env var for the linker optional

* guess the target triple when dx acts as the linker intercept

* automatically find the linker for normal builds

* use the right linker in wasm mode

* fix mac linker

* look inside archive files (rlibs)

* remove out.txt

* fix clippy

* fix typo

* fix clippy

* fix build

* fix dx link name in docs

* ignore libcompiler_builtins warnings

* fix playwright

* pass typocheck

* fix printing sysroot

* Modify final binary

* fix wasm assets

* remove old link section logic

* fix tests

* forward manganis export args

* fail to evaluate relocatable address

* retain all exports in the same pattern as other platforms

* Fix adding assets during a wasm hot patch

* less nesting and remove logs

* fix clippy

* log pdb files

* read from pdb

* print data section contents

* print with object

* normal windows builds working

* fix hot patch windows

* clean up logs

* fix clippy

* clean up assets

* use the most recent pdb

* fix doc test

* fix typo

* tiny nits

* make bundled a reference instead of a pointer

* add helpful comment to impl

* match on file format

* Fix asset hashing comment

---------

Co-authored-by: Jonathan Kelley <jkelleyrtp@gmail.com>

packages/cli-opt/src/build_info.rs

@@ -0,0 +1,10 @@
+// The file has been placed there by the build script.
+include!(concat!(env!("OUT_DIR"), "/built.rs"));
+
+pub(crate) fn version() -> String {
+    format!(
+        "{} ({})",
+        PKG_VERSION,
+        GIT_COMMIT_HASH_SHORT.unwrap_or("was built without git repository")
+    )
+}

packages/cli-opt/src/css.rs

@@ -1,4 +1,4 @@
-use std::path::Path;
+use std::{hash::Hasher, path::Path};
 
 use anyhow::{anyhow, Context};
 use codemap::SpanLoc;
@@ -146,12 +146,11 @@ pub(crate) fn minify_css(css: &str) -> anyhow::Result<String> {
     Ok(res.code)
 }
 
-/// Process an scss/sass file into css.
-pub(crate) fn process_scss(
+/// Compile scss with grass
+pub(crate) fn compile_scss(
     scss_options: &CssAssetOptions,
     source: &Path,
-    output_path: &Path,
-) -> anyhow::Result<()> {
+) -> anyhow::Result<String> {
     let style = match scss_options.minified() {
         true => OutputStyle::Compressed,
         false => OutputStyle::Expanded,
@@ -162,7 +161,18 @@ pub(crate) fn process_scss(
         .quiet(false)
         .logger(&ScssLogger {});
 
-    let css = grass::from_path(source, &options)?;
+    let css = grass::from_path(source, &options)
+        .with_context(|| format!("Failed to compile scss file: {}", source.display()))?;
+    Ok(css)
+}
+
+/// Process an scss/sass file into css.
+pub(crate) fn process_scss(
+    scss_options: &CssAssetOptions,
+    source: &Path,
+    output_path: &Path,
+) -> anyhow::Result<()> {
+    let css = compile_scss(scss_options, source)?;
     let minified = minify_css(&css)?;
 
     std::fs::write(output_path, minified).with_context(|| {
@@ -199,3 +209,19 @@ impl grass::Logger for ScssLogger {
         );
     }
 }
+
+/// Hash the inputs to the scss file
+pub(crate) fn hash_scss(
+    scss_options: &CssAssetOptions,
+    source: &Path,
+    hasher: &mut impl Hasher,
+) -> anyhow::Result<()> {
+    // Grass doesn't expose the ast for us to traverse the imports in the file. Instead of parsing scss ourselves
+    // we just hash the expanded version of the file for now
+    let css = compile_scss(scss_options, source)?;
+
+    // Hash the compiled css
+    hasher.write(css.as_bytes());
+
+    Ok(())
+}

packages/cli-opt/src/file.rs

@@ -1,4 +1,5 @@
 use anyhow::Context;
+use manganis::{CssModuleAssetOptions, FolderAssetOptions};
 use manganis_core::{AssetOptions, CssAssetOptions, ImageAssetOptions, JsAssetOptions};
 use std::path::Path;
 
@@ -33,7 +34,7 @@ pub(crate) fn process_file_to_with_options(
     }
     if let Some(parent) = output_path.parent() {
         if !parent.exists() {
-            std::fs::create_dir_all(parent)?;
+            std::fs::create_dir_all(parent).context("Failed to create directory")?;
         }
     }
 
@@ -47,63 +48,94 @@ pub(crate) fn process_file_to_with_options(
             .unwrap_or_default()
             .to_string_lossy()
     ));
+    let resolved_options = resolve_asset_options(source, options);
 
-    match options {
-        AssetOptions::Unknown => match source.extension().map(|e| e.to_string_lossy()).as_deref() {
-            Some("css") => {
-                process_css(&CssAssetOptions::new(), source, &temp_path)?;
-            }
-            Some("scss" | "sass") => {
-                process_scss(&CssAssetOptions::new(), source, &temp_path)?;
-            }
-            Some("js") => {
-                process_js(&JsAssetOptions::new(), source, &temp_path, !in_folder)?;
-            }
-            Some("json") => {
-                process_json(source, &temp_path)?;
-            }
-            Some("jpg" | "jpeg" | "png" | "webp" | "avif") => {
-                process_image(&ImageAssetOptions::new(), source, &temp_path)?;
-            }
-            Some(_) | None => {
-                if source.is_dir() {
-                    process_folder(source, &temp_path)?;
-                } else {
-                    let source_file = std::fs::File::open(source)?;
-                    let mut reader = std::io::BufReader::new(source_file);
-                    let output_file = std::fs::File::create(&temp_path)?;
-                    let mut writer = std::io::BufWriter::new(output_file);
-                    std::io::copy(&mut reader, &mut writer).with_context(|| {
-                        format!(
-                            "Failed to write file to output location: {}",
-                            temp_path.display()
-                        )
-                    })?;
-                }
-            }
-        },
-        AssetOptions::Css(options) => {
+    match &resolved_options {
+        ResolvedAssetType::Css(options) => {
             process_css(options, source, &temp_path)?;
         }
-        AssetOptions::CssModule(options) => {
+        ResolvedAssetType::CssModule(options) => {
             process_css_module(options, source, output_path, &temp_path)?;
         }
-        AssetOptions::Js(options) => {
+        ResolvedAssetType::Scss(options) => {
+            process_scss(options, source, &temp_path)?;
+        }
+        ResolvedAssetType::Js(options) => {
             process_js(options, source, &temp_path, !in_folder)?;
         }
-        AssetOptions::Image(options) => {
+        ResolvedAssetType::Image(options) => {
             process_image(options, source, &temp_path)?;
         }
-        AssetOptions::Folder(_) => {
+        ResolvedAssetType::Json => {
+            process_json(source, &temp_path)?;
+        }
+        ResolvedAssetType::Folder(_) => {
             process_folder(source, &temp_path)?;
         }
-        _ => {
-            tracing::warn!("Unknown asset options: {:?}", options);
+        ResolvedAssetType::File => {
+            let source_file = std::fs::File::open(source)?;
+            let mut reader = std::io::BufReader::new(source_file);
+            let output_file = std::fs::File::create(&temp_path)?;
+            let mut writer = std::io::BufWriter::new(output_file);
+            std::io::copy(&mut reader, &mut writer).with_context(|| {
+                format!(
+                    "Failed to write file to output location: {}",
+                    temp_path.display()
+                )
+            })?;
         }
     }
 
     // If everything was successful, rename the temp file to the final output path
-    std::fs::rename(temp_path, output_path)?;
+    std::fs::rename(temp_path, output_path).context("Failed to rename output file")?;
 
     Ok(())
 }
+
+pub(crate) enum ResolvedAssetType {
+    /// An image asset
+    Image(ImageAssetOptions),
+    /// A css asset
+    Css(CssAssetOptions),
+    /// A css module asset
+    CssModule(CssModuleAssetOptions),
+    /// A SCSS asset
+    Scss(CssAssetOptions),
+    /// A javascript asset
+    Js(JsAssetOptions),
+    /// A json asset
+    Json,
+    /// A folder asset
+    Folder(FolderAssetOptions),
+    /// A generic file
+    File,
+}
+
+pub(crate) fn resolve_asset_options(source: &Path, options: &AssetOptions) -> ResolvedAssetType {
+    match options {
+        AssetOptions::Image(image) => ResolvedAssetType::Image(*image),
+        AssetOptions::Css(css) => ResolvedAssetType::Css(*css),
+        AssetOptions::CssModule(css) => ResolvedAssetType::CssModule(*css),
+        AssetOptions::Js(js) => ResolvedAssetType::Js(*js),
+        AssetOptions::Folder(folder) => ResolvedAssetType::Folder(*folder),
+        AssetOptions::Unknown => resolve_unknown_asset_options(source),
+        _ => {
+            tracing::warn!("Unknown asset options... you may need to update the Dioxus CLI. Defaulting to a generic file: {:?}", options);
+            resolve_unknown_asset_options(source)
+        }
+    }
+}
+
+fn resolve_unknown_asset_options(source: &Path) -> ResolvedAssetType {
+    match source.extension().map(|e| e.to_string_lossy()).as_deref() {
+        Some("scss" | "sass") => ResolvedAssetType::Scss(CssAssetOptions::new()),
+        Some("css") => ResolvedAssetType::Css(CssAssetOptions::new()),
+        Some("js") => ResolvedAssetType::Js(JsAssetOptions::new()),
+        Some("json") => ResolvedAssetType::Json,
+        Some("jpg" | "jpeg" | "png" | "webp" | "avif") => {
+            ResolvedAssetType::Image(ImageAssetOptions::new())
+        }
+        _ if source.is_dir() => ResolvedAssetType::Folder(FolderAssetOptions::new()),
+        _ => ResolvedAssetType::File,
+    }
+}

packages/cli-opt/src/hash.rs

@@ -0,0 +1,162 @@
+//! Utilities for creating hashed paths to assets in Manganis. This module defines [`AssetHash`] which is used to create a hashed path to an asset in both the CLI and the macro.
+
+use std::{
+    hash::{Hash, Hasher},
+    io::Read,
+    path::{Path, PathBuf},
+};
+
+use crate::{
+    css::hash_scss,
+    file::{resolve_asset_options, ResolvedAssetType},
+    js::hash_js,
+};
+use manganis::{AssetOptions, BundledAsset};
+
+/// The opaque hash type manganis uses to identify assets. Each time an asset or asset options change, this hash will
+/// change. This hash is included in the URL of the bundled asset for cache busting.
+struct AssetHash {
+    /// We use a wrapper type here to hide the exact size of the hash so we can switch to a sha hash in a minor version bump
+    hash: [u8; 8],
+}
+
+impl AssetHash {
+    /// Create a new asset hash
+    const fn new(hash: u64) -> Self {
+        Self {
+            hash: hash.to_le_bytes(),
+        }
+    }
+
+    /// Get the hash bytes
+    pub const fn bytes(&self) -> &[u8] {
+        &self.hash
+    }
+
+    /// Create a new asset hash for a file. The input file to this function should be fully resolved
+    pub fn hash_file_contents(
+        options: &AssetOptions,
+        file_path: impl AsRef<Path>,
+    ) -> anyhow::Result<AssetHash> {
+        hash_file(options, file_path.as_ref())
+    }
+}
+
+/// Process a specific file asset with the given options reading from the source and writing to the output path
+fn hash_file(options: &AssetOptions, source: &Path) -> anyhow::Result<AssetHash> {
+    // Create a hasher
+    let mut hash = std::collections::hash_map::DefaultHasher::new();
+    options.hash(&mut hash);
+    // Hash the version of CLI opt
+    hash.write(crate::build_info::version().as_bytes());
+    hash_file_with_options(options, source, &mut hash, false)?;
+
+    let hash = hash.finish();
+    Ok(AssetHash::new(hash))
+}
+
+/// Process a specific file asset with additional options
+pub(crate) fn hash_file_with_options(
+    options: &AssetOptions,
+    source: &Path,
+    hasher: &mut impl Hasher,
+    in_folder: bool,
+) -> anyhow::Result<()> {
+    let resolved_options = resolve_asset_options(source, options);
+
+    match &resolved_options {
+        // Scss and JS can import files during the bundling process. We need to hash
+        // both the files themselves and any imports they have
+        ResolvedAssetType::Scss(options) => {
+            hash_scss(options, source, hasher)?;
+        }
+        ResolvedAssetType::Js(options) => {
+            hash_js(options, source, hasher, !in_folder)?;
+        }
+
+        // Otherwise, we can just hash the file contents
+        ResolvedAssetType::CssModule(_)
+        | ResolvedAssetType::Css(_)
+        | ResolvedAssetType::Image(_)
+        | ResolvedAssetType::Json
+        | ResolvedAssetType::File => {
+            hash_file_contents(source, hasher)?;
+        }
+        // Or the folder contents recursively
+        ResolvedAssetType::Folder(_) => {
+            let files = std::fs::read_dir(source)?;
+            for file in files.flatten() {
+                let path = file.path();
+                hash_file_with_options(options, &path, hasher, true)?;
+            }
+        }
+    }
+
+    Ok(())
+}
+
+pub(crate) fn hash_file_contents(source: &Path, hasher: &mut impl Hasher) -> anyhow::Result<()> {
+    // Otherwise, open the file to get its contents
+    let mut file = std::fs::File::open(source)?;
+
+    // We add a hash to the end of the file so it is invalidated when the bundled version of the file changes
+    // The hash includes the file contents, the options, and the version of manganis. From the macro, we just
+    // know the file contents, so we only include that hash
+    let mut buffer = [0; 8192];
+    loop {
+        let read = file.read(&mut buffer)?;
+        if read == 0 {
+            break;
+        }
+        hasher.write(&buffer[..read]);
+    }
+    Ok(())
+}
+
+/// Add a hash to the asset, or log an error if it fails
+pub fn add_hash_to_asset(asset: &mut BundledAsset) {
+    let source = asset.absolute_source_path();
+    match AssetHash::hash_file_contents(asset.options(), source) {
+        Ok(hash) => {
+            let options = *asset.options();
+
+            // Set the bundled path to the source path with the hash appended before the extension
+            let source_path = PathBuf::from(source);
+            let Some(file_name) = source_path.file_name() else {
+                tracing::error!("Failed to get file name from path: {source}");
+                return;
+            };
+            // The output extension path is the extension set by the options
+            // or the extension of the source file if we don't recognize the file
+            let mut ext = asset.options().extension().map(Into::into).or_else(|| {
+                source_path
+                    .extension()
+                    .map(|ext| ext.to_string_lossy().to_string())
+            });
+
+            // Rewrite scss as css
+            if let Some("scss" | "sass") = ext.as_deref() {
+                ext = Some("css".to_string());
+            }
+
+            let hash = hash.bytes();
+            let hash = hash
+                .iter()
+                .map(|byte| format!("{byte:x}"))
+                .collect::<String>();
+            let file_stem = source_path.file_stem().unwrap_or(file_name);
+            let mut bundled_path = PathBuf::from(format!("{}-{hash}", file_stem.to_string_lossy()));
+
+            if let Some(ext) = ext {
+                bundled_path.set_extension(ext);
+            }
+
+            let bundled_path = bundled_path.to_string_lossy().to_string();
+
+            *asset = BundledAsset::new(source, &bundled_path, options);
+        }
+        Err(err) => {
+            tracing::error!("Failed to hash asset: {err}");
+        }
+    }
+}

packages/cli-opt/src/image/mod.rs

@@ -14,7 +14,8 @@ pub(crate) fn process_image(
     output_path: &Path,
 ) -> anyhow::Result<()> {
     let mut image = image::ImageReader::new(std::io::Cursor::new(&*std::fs::read(source)?))
-        .with_guessed_format()?
+        .with_guessed_format()
+        .context("Failed to guess image format")?
         .decode();
 
     if let Ok(image) = &mut image {
@@ -25,10 +26,10 @@ pub(crate) fn process_image(
 
     match (image, image_options.format()) {
         (image, ImageFormat::Png) => {
-            compress_png(image?, output_path);
+            compress_png(image.context("Failed to decode image")?, output_path);
         }
         (image, ImageFormat::Jpg) => {
-            compress_jpg(image?, output_path)?;
+            compress_jpg(image.context("Failed to decode image")?, output_path)?;
         }
         (Ok(image), ImageFormat::Avif) => {
             if let Err(error) = image.save(output_path) {
@@ -41,21 +42,31 @@ pub(crate) fn process_image(
             }
         }
         (Ok(image), _) => {
-            image.save(output_path)?;
-        }
-        // If we can't decode the image or it is of an unknown type, we just copy the file
-        _ => {
-            let source_file = std::fs::File::open(source)?;
-            let mut reader = std::io::BufReader::new(source_file);
-            let output_file = std::fs::File::create(output_path)?;
-            let mut writer = std::io::BufWriter::new(output_file);
-            std::io::copy(&mut reader, &mut writer).with_context(|| {
+            image.save(output_path).with_context(|| {
                 format!(
-                    "Failed to write image to output location: {}",
+                    "Failed to save image (from {}) with path {}",
+                    source.display(),
                     output_path.display()
                 )
             })?;
         }
+        // If we can't decode the image or it is of an unknown type, we just copy the file
+        _ => {
+            let source_file = std::fs::File::open(source).context("Failed to open source file")?;
+            let mut reader = std::io::BufReader::new(source_file);
+            let output_file = std::fs::File::create(output_path).with_context(|| {
+                format!("Failed to create output file: {}", output_path.display())
+            })?;
+            let mut writer = std::io::BufWriter::new(output_file);
+            std::io::copy(&mut reader, &mut writer)
+                .with_context(|| {
+                    format!(
+                        "Failed to write image to output location: {}",
+                        output_path.display()
+                    )
+                })
+                .context("Failed to copy image data")?;
+        }
     }
 
     Ok(())

packages/cli-opt/src/js.rs

@@ -1,3 +1,4 @@
+use std::hash::Hasher;
 use std::path::Path;
 use std::path::PathBuf;
 
@@ -24,6 +25,8 @@ use swc_ecma_codegen::text_writer::JsWriter;
 use swc_ecma_loader::{resolvers::node::NodeModulesResolver, TargetEnv};
 use swc_ecma_parser::{parse_file_as_module, Syntax};
 
+use crate::hash::hash_file_contents;
+
 struct TracingEmitter;
 
 impl Emitter for TracingEmitter {
@@ -43,30 +46,32 @@ impl Emitter for TracingEmitter {
     }
 }
 
+/// Run a closure with the swc globals and handler set up
+fn inside_handler<O>(f: impl FnOnce(&Globals, Lrc<SourceMap>) -> O) -> O {
+    let globals = Globals::new();
+    let cm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
+    let handler = Handler::with_emitter_and_flags(Box::new(TracingEmitter), Default::default());
+    GLOBALS.set(&globals, || HANDLER.set(&handler, || f(&globals, cm)))
+}
+
 fn bundle_js_to_writer(
     file: PathBuf,
     bundle: bool,
     minify: bool,
     write_to: &mut impl std::io::Write,
 ) -> anyhow::Result<()> {
-    let globals = Globals::new();
-    let handler = Handler::with_emitter_and_flags(Box::new(TracingEmitter), Default::default());
-    GLOBALS.set(&globals, || {
-        HANDLER.set(&handler, || {
-            bundle_js_to_writer_inside_handler(&globals, file, bundle, minify, write_to)
-        })
+    inside_handler(|globals, cm| {
+        bundle_js_to_writer_inside_handler(globals, cm, file, bundle, minify, write_to)
     })
 }
 
-fn bundle_js_to_writer_inside_handler(
+fn resolve_js_inside_handler(
     globals: &Globals,
     file: PathBuf,
     bundle: bool,
-    minify: bool,
-    write_to: &mut impl std::io::Write,
-) -> anyhow::Result<()> {
-    let cm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
-    let mut module = if bundle {
+    cm: &Lrc<SourceMap>,
+) -> anyhow::Result<Module> {
+    if bundle {
         let node_resolver = NodeModulesResolver::new(TargetEnv::Browser, Default::default(), true);
         let mut bundler = Bundler::new(
             globals,
@@ -89,7 +94,7 @@ fn bundle_js_to_writer_inside_handler(
         let bundle = bundles
             .pop()
             .ok_or_else(|| anyhow::anyhow!("swc did not output any bundles"))?;
-        bundle.module
+        Ok(bundle.module)
     } else {
         let fm = cm.load_file(Path::new(&file)).expect("Failed to load file");
 
@@ -108,8 +113,19 @@ fn bundle_js_to_writer_inside_handler(
                 error.cancel();
                 anyhow::anyhow!("{}", error.message())
             })
-        })?
-    };
+        })
+    }
+}
+
+fn bundle_js_to_writer_inside_handler(
+    globals: &Globals,
+    cm: Lrc<SourceMap>,
+    file: PathBuf,
+    bundle: bool,
+    minify: bool,
+    write_to: &mut impl std::io::Write,
+) -> anyhow::Result<()> {
+    let mut module = resolve_js_inside_handler(globals, file, bundle, &cm)?;
 
     if minify {
         module = swc_ecma_minifier::optimize(
@@ -246,3 +262,34 @@ pub(crate) fn process_js(
 
     Ok(())
 }
+
+fn hash_js_module(file: PathBuf, hasher: &mut impl Hasher, bundle: bool) -> anyhow::Result<()> {
+    inside_handler(|globals, cm| {
+        _ = resolve_js_inside_handler(globals, file, bundle, &cm)?;
+
+        for file in cm.files().iter() {
+            let hash = file.src_hash;
+            hasher.write(&hash.to_le_bytes());
+        }
+
+        Ok(())
+    })
+}
+
+pub(crate) fn hash_js(
+    js_options: &JsAssetOptions,
+    source: &Path,
+    hasher: &mut impl Hasher,
+    bundle: bool,
+) -> anyhow::Result<()> {
+    if js_options.minified() {
+        if let Err(err) = hash_js_module(source.to_path_buf(), hasher, bundle) {
+            tracing::error!("Failed to minify js. Falling back to non-minified: {err}");
+            hash_file_contents(source, hasher)?;
+        }
+    } else {
+        hash_file_contents(source, hasher)?;
+    }
+
+    Ok(())
+}

packages/cli-opt/src/lib.rs

@@ -1,19 +1,23 @@
 use anyhow::Context;
-use manganis_core::linker::LinkSection;
+use manganis::AssetOptions;
 use manganis_core::BundledAsset;
-use object::{read::archive::ArchiveFile, File as ObjectFile, Object, ObjectSection};
+use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
 use serde::{Deserialize, Serialize};
-use std::path::Path;
-use std::{collections::HashMap, path::PathBuf};
+use std::collections::{HashMap, HashSet};
+use std::path::{Path, PathBuf};
+use std::sync::{Arc, RwLock};
 
+mod build_info;
 mod css;
 mod file;
 mod folder;
+mod hash;
 mod image;
 mod js;
 mod json;
 
 pub use file::process_file_to;
+pub use hash::add_hash_to_asset;
 
 /// A manifest of all assets collected from dependencies
 ///
@@ -21,7 +25,7 @@ pub use file::process_file_to;
 #[derive(Debug, PartialEq, Default, Clone, Serialize, Deserialize)]
 pub struct AssetManifest {
     /// Map of bundled asset name to the asset itself
-    pub assets: HashMap<PathBuf, BundledAsset>,
+    assets: HashMap<PathBuf, HashSet<BundledAsset>>,
 }
 
 impl AssetManifest {
@@ -31,100 +35,82 @@ impl AssetManifest {
         asset_path: &Path,
         options: manganis::AssetOptions,
     ) -> anyhow::Result<BundledAsset> {
-        let hash = manganis_core::hash::AssetHash::hash_file_contents(asset_path)
-            .context("Failed to hash file")?;
-
         let output_path_str = asset_path.to_str().ok_or(anyhow::anyhow!(
             "Failed to convert wasm bindgen output path to string"
         ))?;
 
-        let bundled_asset =
-            manganis::macro_helpers::create_bundled_asset(output_path_str, hash.bytes(), options);
+        let mut bundled_asset =
+            manganis::macro_helpers::create_bundled_asset(output_path_str, options);
+        add_hash_to_asset(&mut bundled_asset);
 
-        self.assets.insert(asset_path.into(), bundled_asset);
+        self.assets
+            .entry(asset_path.to_path_buf())
+            .or_default()
+            .insert(bundled_asset);
 
         Ok(bundled_asset)
     }
 
-    #[allow(dead_code)]
+    /// Insert an existing bundled asset to the manifest
+    pub fn insert_asset(&mut self, asset: BundledAsset) {
+        let asset_path = asset.absolute_source_path();
+        self.assets
+            .entry(asset_path.into())
+            .or_default()
+            .insert(asset);
+    }
+
+    /// Get any assets that are tied to a specific source file
+    pub fn get_assets_for_source(&self, path: &Path) -> Option<&HashSet<BundledAsset>> {
+        self.assets.get(path)
+    }
+
+    /// Check if the manifest contains a specific asset
+    pub fn contains(&self, asset: &BundledAsset) -> bool {
+        self.assets
+            .get(&PathBuf::from(asset.absolute_source_path()))
+            .is_some_and(|assets| assets.contains(asset))
+    }
+
+    /// Iterate over all the assets in the manifest
+    pub fn assets(&self) -> impl Iterator<Item = &BundledAsset> {
+        self.assets.values().flat_map(|assets| assets.iter())
+    }
+
     pub fn load_from_file(path: &Path) -> anyhow::Result<Self> {
         let src = std::fs::read_to_string(path)?;
 
         serde_json::from_str(&src)
             .with_context(|| format!("Failed to parse asset manifest from {path:?}\n{src}"))
     }
-
-    /// Fill this manifest with a file object/rlib files, typically extracted from the linker intercepted
-    pub fn add_from_object_path(&mut self, path: &Path) -> anyhow::Result<()> {
-        let data = std::fs::read(path)?;
-
-        match path.extension().and_then(|ext| ext.to_str()) {
-            // Parse an rlib as a collection of objects
-            Some("rlib") => {
-                if let Ok(archive) = object::read::archive::ArchiveFile::parse(&*data) {
-                    self.add_from_archive_file(&archive, &data)?;
-                }
-            }
-            _ => {
-                if let Ok(object) = object::File::parse(&*data) {
-                    self.add_from_object_file(&object)?;
-                }
-            }
-        }
-
-        Ok(())
-    }
-
-    /// Fill this manifest from an rlib / ar file that contains many object files and their entries
-    fn add_from_archive_file(&mut self, archive: &ArchiveFile, data: &[u8]) -> object::Result<()> {
-        // Look through each archive member for object files.
-        // Read the archive member's binary data (we know it's an object file)
-        // And parse it with the normal `object::File::parse` to find the manganis string.
-        for member in archive.members() {
-            let member = member?;
-            let name = String::from_utf8_lossy(member.name()).to_string();
-
-            // Check if the archive member is an object file and parse it.
-            if name.ends_with(".o") {
-                let data = member.data(data)?;
-                let object = object::File::parse(data)?;
-                _ = self.add_from_object_file(&object);
+}
+
+/// Optimize a list of assets in parallel
+pub fn optimize_all_assets(
+    assets_to_transfer: Vec<(PathBuf, PathBuf, AssetOptions)>,
+    on_optimization_start: impl FnMut(&Path, &Path, &AssetOptions) + Sync + Send,
+    on_optimization_end: impl FnMut(&Path, &Path, &AssetOptions) + Sync + Send,
+) -> anyhow::Result<()> {
+    let on_optimization_start = Arc::new(RwLock::new(on_optimization_start));
+    let on_optimization_end = Arc::new(RwLock::new(on_optimization_end));
+    assets_to_transfer
+        .par_iter()
+        .try_for_each(|(from, to, options)| {
+            {
+                let mut on_optimization_start = on_optimization_start.write().unwrap();
+                on_optimization_start(from, to, options);
             }
-        }
-
-        Ok(())
-    }
 
-    /// Fill this manifest with whatever tables might come from the object file
-    fn add_from_object_file(&mut self, obj: &ObjectFile) -> anyhow::Result<()> {
-        for section in obj.sections() {
-            let Ok(section_name) = section.name() else {
-                continue;
-            };
-
-            // Check if the link section matches the asset section for one of the platforms we support. This may not be the current platform if the user is cross compiling
-            let matches = LinkSection::ALL
-                .iter()
-                .any(|x| x.link_section == section_name);
-
-            if !matches {
-                continue;
+            let res = process_file_to(options, from, to);
+            if let Err(err) = res.as_ref() {
+                tracing::error!("Failed to copy asset {from:?}: {err}");
             }
-
-            let bytes = section
-                .uncompressed_data()
-                .context("Could not read uncompressed data from object file")?;
 
-            let mut buffer = const_serialize::ConstReadBuffer::new(&bytes);
-            while let Some((remaining_buffer, asset)) =
-                const_serialize::deserialize_const!(BundledAsset, buffer)
             {
-                self.assets
-                    .insert(asset.absolute_source_path().into(), asset);
-                buffer = remaining_buffer;
+                let mut on_optimization_end = on_optimization_end.write().unwrap();
+                on_optimization_end(from, to, options);
             }
-        }
 
-        Ok(())
-    }
+            res.map(|_| ())
+        })
 }