swaits/dioxus
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move asset hashing into the CLI (#3988)

Browse Source 
* Fix deduping assets

* hash assets in the linker

* fix extracting assets

* Fix the hashed asset name

* deserialize the asset back from the link section

* prevent the read from being inlined

* remove some extra logs

* Fix liveview routing

* Only look for direct dependencies on dioxus features

* fix wasm split test

* On wasm, create a new object file with the asset information

* Fix multiple assets

* add a regression test for 3467

* fix image link

* fix linker out file

* fix linux linker args

* add windows linker command

* fix linux link args

* fix windows target triple

* Expose an option to build assets in the linker intercept

* Fix the standalone optimizer and document it in the dx help command

* include the cli opt version in the hash

* fix clippy

* fix ssg port

* add a safety note to the read_volatile call

* move found assets log

* fix some merge conflicts

* fix build

* make most env var for the linker optional

* guess the target triple when dx acts as the linker intercept

* automatically find the linker for normal builds

* use the right linker in wasm mode

* fix mac linker

* look inside archive files (rlibs)

* remove out.txt

* fix clippy

* fix typo

* fix clippy

* fix build

* fix dx link name in docs

* ignore libcompiler_builtins warnings

* fix playwright

* pass typocheck

* fix printing sysroot

* Modify final binary

* fix wasm assets

* remove old link section logic

* fix tests

* forward manganis export args

* fail to evaluate relocatable address

* retain all exports in the same pattern as other platforms

* Fix adding assets during a wasm hot patch

* less nesting and remove logs

* fix clippy

* log pdb files

* read from pdb

* print data section contents

* print with object

* normal windows builds working

* fix hot patch windows

* clean up logs

* fix clippy

* clean up assets

* use the most recent pdb

* fix doc test

* fix typo

* tiny nits

* make bundled a reference instead of a pointer

* add helpful comment to impl

* match on file format

* Fix asset hashing comment

---------

Co-authored-by: Jonathan Kelley <jkelleyrtp@gmail.com>
This commit is contained in:
Evan AlmloffandJonathan Kelley authored and GitHub committed 2025-05-31 21:45:12 +02:00
1 parent f865e3f559
commit 58669f892d
45 files changed
+1174 -762
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

packages/cli-opt/src/lib.rs
+65 -79
View File
@@ -1,19 +1,23 @@
use anyhow::Context;
use manganis_core::linker::LinkSection;
use manganis::AssetOptions;
use manganis_core::BundledAsset;
use object::{read::archive::ArchiveFile, File as ObjectFile, Object, ObjectSection};
use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
use serde::{Deserialize, Serialize};
use std::path::Path;
use std::{collections::HashMap, path::PathBuf};
use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf};
use std::sync::{Arc, RwLock};
mod build_info;
mod css;
mod file;
mod folder;
mod hash;
mod image;
mod js;
mod json;
pub use file::process_file_to;
pub use hash::add_hash_to_asset;
/// A manifest of all assets collected from dependencies
///
@@ -21,7 +25,7 @@ pub use file::process_file_to;
#[derive(Debug, PartialEq, Default, Clone, Serialize, Deserialize)]
pub struct AssetManifest {
/// Map of bundled asset name to the asset itself
pub assets: HashMap<PathBuf, BundledAsset>,
assets: HashMap<PathBuf, HashSet<BundledAsset>>,
}
impl AssetManifest {
@@ -31,100 +35,82 @@ impl AssetManifest {
asset_path: &Path,
options: manganis::AssetOptions,
) -> anyhow::Result<BundledAsset> {
let hash = manganis_core::hash::AssetHash::hash_file_contents(asset_path)
.context("Failed to hash file")?;
let output_path_str = asset_path.to_str().ok_or(anyhow::anyhow!(
"Failed to convert wasm bindgen output path to string"
))?;
let bundled_asset =
manganis::macro_helpers::create_bundled_asset(output_path_str, hash.bytes(), options);
let mut bundled_asset =
manganis::macro_helpers::create_bundled_asset(output_path_str, options);
add_hash_to_asset(&mut bundled_asset);
self.assets.insert(asset_path.into(), bundled_asset);
self.assets
.entry(asset_path.to_path_buf())
.or_default()
.insert(bundled_asset);
Ok(bundled_asset)
}
#[allow(dead_code)]
/// Insert an existing bundled asset to the manifest
pub fn insert_asset(&mut self, asset: BundledAsset) {
let asset_path = asset.absolute_source_path();
self.assets
.entry(asset_path.into())
.or_default()
.insert(asset);
}
/// Get any assets that are tied to a specific source file
pub fn get_assets_for_source(&self, path: &Path) -> Option<&HashSet<BundledAsset>> {
self.assets.get(path)
}
/// Check if the manifest contains a specific asset
pub fn contains(&self, asset: &BundledAsset) -> bool {
self.assets
.get(&PathBuf::from(asset.absolute_source_path()))
.is_some_and(|assets| assets.contains(asset))
}
/// Iterate over all the assets in the manifest
pub fn assets(&self) -> impl Iterator<Item = &BundledAsset> {
self.assets.values().flat_map(|assets| assets.iter())
}
pub fn load_from_file(path: &Path) -> anyhow::Result<Self> {
let src = std::fs::read_to_string(path)?;
serde_json::from_str(&src)
.with_context(|| format!("Failed to parse asset manifest from {path:?}\n{src}"))
}
/// Fill this manifest with a file object/rlib files, typically extracted from the linker intercepted
pub fn add_from_object_path(&mut self, path: &Path) -> anyhow::Result<()> {
let data = std::fs::read(path)?;
match path.extension().and_then(|ext| ext.to_str()) {
// Parse an rlib as a collection of objects
Some("rlib") => {
if let Ok(archive) = object::read::archive::ArchiveFile::parse(&*data) {
self.add_from_archive_file(&archive, &data)?;
}
}
_ => {
if let Ok(object) = object::File::parse(&*data) {
self.add_from_object_file(&object)?;
}
}
}
Ok(())
}
/// Fill this manifest from an rlib / ar file that contains many object files and their entries
fn add_from_archive_file(&mut self, archive: &ArchiveFile, data: &[u8]) -> object::Result<()> {
// Look through each archive member for object files.
// Read the archive member's binary data (we know it's an object file)
// And parse it with the normal `object::File::parse` to find the manganis string.
for member in archive.members() {
let member = member?;
let name = String::from_utf8_lossy(member.name()).to_string();
// Check if the archive member is an object file and parse it.
if name.ends_with(".o") {
let data = member.data(data)?;
let object = object::File::parse(data)?;
_ = self.add_from_object_file(&object);
}
/// Optimize a list of assets in parallel
pub fn optimize_all_assets(
assets_to_transfer: Vec<(PathBuf, PathBuf, AssetOptions)>,
on_optimization_start: impl FnMut(&Path, &Path, &AssetOptions) + Sync + Send,
on_optimization_end: impl FnMut(&Path, &Path, &AssetOptions) + Sync + Send,
) -> anyhow::Result<()> {
let on_optimization_start = Arc::new(RwLock::new(on_optimization_start));
let on_optimization_end = Arc::new(RwLock::new(on_optimization_end));
assets_to_transfer
.par_iter()
.try_for_each(|(from, to, options)| {
{
let mut on_optimization_start = on_optimization_start.write().unwrap();
on_optimization_start(from, to, options);
}
}
Ok(())
}
/// Fill this manifest with whatever tables might come from the object file
fn add_from_object_file(&mut self, obj: &ObjectFile) -> anyhow::Result<()> {
for section in obj.sections() {
let Ok(section_name) = section.name() else {
continue;
};
// Check if the link section matches the asset section for one of the platforms we support. This may not be the current platform if the user is cross compiling
let matches = LinkSection::ALL
.iter()
.any(|x| x.link_section == section_name);
if !matches {
continue;
let res = process_file_to(options, from, to);
if let Err(err) = res.as_ref() {
tracing::error!("Failed to copy asset {from:?}: {err}");
}
let bytes = section
.uncompressed_data()
.context("Could not read uncompressed data from object file")?;
let mut buffer = const_serialize::ConstReadBuffer::new(&bytes);
while let Some((remaining_buffer, asset)) =
const_serialize::deserialize_const!(BundledAsset, buffer)
{
self.assets
.insert(asset.absolute_source_path().into(), asset);
buffer = remaining_buffer;
let mut on_optimization_end = on_optimization_end.write().unwrap();
on_optimization_end(from, to, options);
}
}
Ok(())
}
res.map(|_| ())
})
}