swaits/dioxus
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move asset hashing into the CLI (#3988)

Browse Source 
* Fix deduping assets

* hash assets in the linker

* fix extracting assets

* Fix the hashed asset name

* deserialize the asset back from the link section

* prevent the read from being inlined

* remove some extra logs

* Fix liveview routing

* Only look for direct dependencies on dioxus features

* fix wasm split test

* On wasm, create a new object file with the asset information

* Fix multiple assets

* add a regression test for 3467

* fix image link

* fix linker out file

* fix linux linker args

* add windows linker command

* fix linux link args

* fix windows target triple

* Expose an option to build assets in the linker intercept

* Fix the standalone optimizer and document it in the dx help command

* include the cli opt version in the hash

* fix clippy

* fix ssg port

* add a safety note to the read_volatile call

* move found assets log

* fix some merge conflicts

* fix build

* make most env var for the linker optional

* guess the target triple when dx acts as the linker intercept

* automatically find the linker for normal builds

* use the right linker in wasm mode

* fix mac linker

* look inside archive files (rlibs)

* remove out.txt

* fix clippy

* fix typo

* fix clippy

* fix build

* fix dx link name in docs

* ignore libcompiler_builtins warnings

* fix playwright

* pass typocheck

* fix printing sysroot

* Modify final binary

* fix wasm assets

* remove old link section logic

* fix tests

* forward manganis export args

* fail to evaluate relocatable address

* retain all exports in the same pattern as other platforms

* Fix adding assets during a wasm hot patch

* less nesting and remove logs

* fix clippy

* log pdb files

* read from pdb

* print data section contents

* print with object

* normal windows builds working

* fix hot patch windows

* clean up logs

* fix clippy

* clean up assets

* use the most recent pdb

* fix doc test

* fix typo

* tiny nits

* make bundled a reference instead of a pointer

* add helpful comment to impl

* match on file format

* Fix asset hashing comment

---------

Co-authored-by: Jonathan Kelley <jkelleyrtp@gmail.com>
This commit is contained in:
Evan AlmloffandJonathan Kelley authored and GitHub committed 2025-05-31 21:45:12 +02:00
1 parent f865e3f559
commit 58669f892d
45 files changed
+1174 -762
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

packages/manganis/manganis/src/hash.rs
-39
View File
@@ -1,39 +0,0 @@
use const_serialize::{serialize_const, ConstVec, SerializeConst};
// From rustchash - https://github.com/rust-lang/rustc-hash/blob/6745258da00b7251bed4a8461871522d0231a9c7/src/lib.rs#L98
const K: u64 = 0xf1357aea2e62a9c5;
pub(crate) struct ConstHasher {
hash: u64,
}
impl ConstHasher {
pub const fn new() -> Self {
Self { hash: 0 }
}
pub const fn finish(&self) -> u64 {
self.hash
}
pub const fn write(mut self, bytes: &[u8]) -> Self {
let mut i = 0;
while i < bytes.len() {
self = self.write_byte(bytes[i]);
i += 1;
}
self
}
pub const fn write_byte(mut self, byte: u8) -> Self {
self.hash = self.hash.wrapping_add(byte as u64).wrapping_mul(K);
self
}
pub const fn hash_by_bytes<T: SerializeConst>(self, item: &T) -> Self {
let mut bytes = ConstVec::new();
bytes = serialize_const(item, bytes);
let bytes = bytes.as_ref();
self.write(bytes)
}
}
packages/manganis/manganis/src/lib.rs
-1
View File
@@ -1,7 +1,6 @@
#![doc = include_str!("../README.md")]
#![deny(missing_docs)]
mod hash;
#[doc(hidden)]
pub mod macro_helpers;
pub use manganis_macro::{asset, css_module};
packages/manganis/manganis/src/macro_helpers.rs
+22 -201
View File
@@ -1,17 +1,13 @@
pub use const_serialize;
use const_serialize::{serialize_const, ConstStr, ConstVec};
use const_serialize::{serialize_const, ConstStr, ConstVec, SerializeConst};
use manganis_core::{AssetOptions, BundledAsset};
use crate::hash::ConstHasher;
const PLACEHOLDER_HASH: ConstStr =
ConstStr::new("this is a placeholder path which will be replaced by the linker");
/// Create a bundled asset from the input path, the content hash, and the asset options
pub const fn create_bundled_asset(
input_path: &str,
content_hash: &[u8],
asset_config: AssetOptions,
) -> BundledAsset {
let hashed_path = generate_unique_path_with_byte_hash(input_path, content_hash, &asset_config);
BundledAsset::new_from_const(ConstStr::new(input_path), hashed_path, asset_config)
pub const fn create_bundled_asset(input_path: &str, asset_config: AssetOptions) -> BundledAsset {
BundledAsset::new_from_const(ConstStr::new(input_path), PLACEHOLDER_HASH, asset_config)
}
/// Create a bundled asset from the input path, the content hash, and the asset options with a relative asset deprecation warning
@@ -22,204 +18,29 @@ pub const fn create_bundled_asset(
)]
pub const fn create_bundled_asset_relative(
input_path: &str,
content_hash: &[u8],
asset_config: AssetOptions,
) -> BundledAsset {
create_bundled_asset(input_path, content_hash, asset_config)
}
/// Format the input path with a hash to create an unique output path for the macro in the form `{input_path}-{hash}.{extension}`
pub const fn generate_unique_path(
input_path: &str,
content_hash: u64,
asset_config: &AssetOptions,
) -> ConstStr {
let byte_hash = content_hash.to_le_bytes();
generate_unique_path_with_byte_hash(input_path, &byte_hash, asset_config)
}
/// Format the input path with a hash to create an unique output path for the macro in the form `{input_path}-{hash}.{extension}`
const fn generate_unique_path_with_byte_hash(
input_path: &str,
content_hash: &[u8],
asset_config: &AssetOptions,
) -> ConstStr {
// Format the unique path with the format `{input_path}-{hash}.{extension}`
// Start with the input path
let mut input_path = ConstStr::new(input_path);
// Then strip the prefix from the input path. The path comes from the build platform, but
// in wasm, we don't know what the path separator is from the build platform. We need to
// split by both unix and windows paths and take the smallest one
let mut extension = None;
match (input_path.rsplit_once('/'), input_path.rsplit_once('\\')) {
(Some((_, unix_new_input_path)), Some((_, windows_new_input_path))) => {
input_path = if unix_new_input_path.len() < windows_new_input_path.len() {
unix_new_input_path
} else {
windows_new_input_path
};
}
(Some((_, unix_new_input_path)), _) => {
input_path = unix_new_input_path;
}
(_, Some((_, windows_new_input_path))) => {
input_path = windows_new_input_path;
}
_ => {}
}
if let Some((new_input_path, new_extension)) = input_path.rsplit_once('.') {
extension = Some(new_extension);
input_path = new_input_path;
}
// Then add a dash
let mut macro_output_path = input_path.push_str("-");
// Hash the contents along with the asset config to create a unique hash for the asset
// When this hash changes, the client needs to re-fetch the asset
let mut hasher = ConstHasher::new();
hasher = hasher.write(content_hash);
hasher = hasher.hash_by_bytes(asset_config);
let hash = hasher.finish();
// Then add the hash in hex form
let hash_bytes = hash.to_le_bytes();
let mut i = 0;
while i < hash_bytes.len() {
let byte = hash_bytes[i];
let first = byte >> 4;
let second = byte & 0x0f;
const fn byte_to_char(byte: u8) -> char {
match char::from_digit(byte as u32, 16) {
Some(c) => c,
None => panic!("byte must be a valid digit"),
}
}
macro_output_path = macro_output_path.push(byte_to_char(first));
macro_output_path = macro_output_path.push(byte_to_char(second));
i += 1;
}
// Finally add the extension
match asset_config.extension() {
Some(extension) => {
macro_output_path = macro_output_path.push('.');
macro_output_path = macro_output_path.push_str(extension)
}
None => {
if let Some(extension) = extension {
macro_output_path = macro_output_path.push('.');
let ext_bytes = extension.as_str().as_bytes();
// Rewrite scss as css
if bytes_equal(ext_bytes, b"scss") || bytes_equal(ext_bytes, b"sass") {
macro_output_path = macro_output_path.push_str("css")
} else {
macro_output_path = macro_output_path.push_str(extension.as_str())
}
}
}
}
macro_output_path
}
/// Construct the hash used by manganis and cli-opt to uniquely identify a asset based on its contents
pub const fn hash_asset(asset_config: &AssetOptions, content_hash: u64) -> ConstStr {
let mut string = ConstStr::new("");
// Hash the contents along with the asset config to create a unique hash for the asset
// When this hash changes, the client needs to re-fetch the asset
let mut hasher = ConstHasher::new();
hasher = hasher.write(&content_hash.to_le_bytes());
hasher = hasher.hash_by_bytes(asset_config);
let hash = hasher.finish();
// Then add the hash in hex form
let hash_bytes = hash.to_le_bytes();
let mut i = 0;
while i < hash_bytes.len() {
let byte = hash_bytes[i];
let first = byte >> 4;
let second = byte & 0x0f;
const fn byte_to_char(byte: u8) -> char {
match char::from_digit(byte as u32, 16) {
Some(c) => c,
None => panic!("byte must be a valid digit"),
}
}
string = string.push(byte_to_char(first));
string = string.push(byte_to_char(second));
i += 1;
}
string
}
const fn bytes_equal(left: &[u8], right: &[u8]) -> bool {
if left.len() != right.len() {
return false;
}
let mut i = 0;
while i < left.len() {
if left[i] != right[i] {
return false;
}
i += 1;
}
true
}
#[test]
fn test_unique_path() {
use manganis_core::{ImageAssetOptions, ImageFormat};
use std::path::PathBuf;
let mut input_path = PathBuf::from("some");
input_path.push("prefix");
input_path.push("test.png");
let content_hash = 123456789;
let asset_config = AssetOptions::Image(ImageAssetOptions::new().with_format(ImageFormat::Avif));
let output_path =
generate_unique_path(&input_path.to_string_lossy(), content_hash, &asset_config);
assert_eq!(output_path.as_str(), "test-603a88fe296462a3.avif");
// Changing the path without changing the contents shouldn't change the hash
let mut input_path = PathBuf::from("some");
input_path.push("prefix");
input_path.push("prefix");
input_path.push("test.png");
let content_hash = 123456789;
let asset_config = AssetOptions::Image(ImageAssetOptions::new().with_format(ImageFormat::Avif));
let output_path =
generate_unique_path(&input_path.to_string_lossy(), content_hash, &asset_config);
assert_eq!(output_path.as_str(), "test-603a88fe296462a3.avif");
let mut input_path = PathBuf::from("test");
input_path.push("ing");
input_path.push("test");
let content_hash = 123456789;
let asset_config = AssetOptions::Unknown;
let output_path =
generate_unique_path(&input_path.to_string_lossy(), content_hash, &asset_config);
assert_eq!(output_path.as_str(), "test-8d6e32dc0b45f853");
// Just changing the content hash should change the total hash
let mut input_path = PathBuf::from("test");
input_path.push("ing");
input_path.push("test");
let content_hash = 123456780;
let asset_config = AssetOptions::Unknown;
let output_path =
generate_unique_path(&input_path.to_string_lossy(), content_hash, &asset_config);
assert_eq!(output_path.as_str(), "test-40783366737abc4d");
create_bundled_asset(input_path, asset_config)
}
/// Serialize an asset to a const buffer
pub const fn serialize_asset(asset: &BundledAsset) -> ConstVec<u8> {
let write = ConstVec::new();
serialize_const(asset, write)
let data = ConstVec::new();
let mut data = serialize_const(asset, data);
// Reserve the maximum size of the asset
while data.len() < BundledAsset::MEMORY_LAYOUT.size() {
data = data.push(0);
}
data
}
/// Deserialize a const buffer into a BundledAsset
pub const fn deserialize_asset(bytes: &[u8]) -> BundledAsset {
let bytes = ConstVec::new().extend(bytes);
match const_serialize::deserialize_const!(BundledAsset, bytes.read()) {
Some((_, asset)) => asset,
None => panic!("Failed to deserialize asset. This may be caused by a mismatch between your dioxus and dioxus-cli versions"),
}
}
/// Copy a slice into a constant sized buffer at compile time