Move asset hashing into the CLI (#3988)

* Fix deduping assets

* hash assets in the linker

* fix extracting assets

* Fix the hashed asset name

* deserialize the asset back from the link section

* prevent the read from being inlined

* remove some extra logs

* Fix liveview routing

* Only look for direct dependencies on dioxus features

* fix wasm split test

* On wasm, create a new object file with the asset information

* Fix multiple assets

* add a regression test for 3467

* fix image link

* fix linker out file

* fix linux linker args

* add windows linker command

* fix linux link args

* fix windows target triple

* Expose an option to build assets in the linker intercept

* Fix the standalone optimizer and document it in the dx help command

* include the cli opt version in the hash

* fix clippy

* fix ssg port

* add a safety note to the read_volatile call

* move found assets log

* fix some merge conflicts

* fix build

* make most env var for the linker optional

* guess the target triple when dx acts as the linker intercept

* automatically find the linker for normal builds

* use the right linker in wasm mode

* fix mac linker

* look inside archive files (rlibs)

* remove out.txt

* fix clippy

* fix typo

* fix clippy

* fix build

* fix dx link name in docs

* ignore libcompiler_builtins warnings

* fix playwright

* pass typocheck

* fix printing sysroot

* Modify final binary

* fix wasm assets

* remove old link section logic

* fix tests

* forward manganis export args

* fail to evaluate relocatable address

* retain all exports in the same pattern as other platforms

* Fix adding assets during a wasm hot patch

* less nesting and remove logs

* fix clippy

* log pdb files

* read from pdb

* print data section contents

* print with object

* normal windows builds working

* fix hot patch windows

* clean up logs

* fix clippy

* clean up assets

* use the most recent pdb

* fix doc test

* fix typo

* tiny nits

* make bundled a reference instead of a pointer

* add helpful comment to impl

* match on file format

* Fix asset hashing comment

---------

Co-authored-by: Jonathan Kelley <jkelleyrtp@gmail.com>

packages/manganis/manganis-core/src/asset.rs

@@ -1,23 +1,13 @@
 use crate::AssetOptions;
-use const_serialize::{ConstStr, SerializeConst};
-use std::path::PathBuf;
+use const_serialize::{deserialize_const, ConstStr, ConstVec, SerializeConst};
+use std::{fmt::Debug, hash::Hash, path::PathBuf};
 
 /// An asset that should be copied by the bundler with some options. This type will be
-/// serialized into the binary and added to the link section [`LinkSection::CURRENT`](crate::linker::LinkSection::CURRENT).
+/// serialized into the binary.
 /// CLIs that support manganis, should pull out the assets from the link section, optimize,
 /// and write them to the filesystem at [`BundledAsset::bundled_path`] for the application
 /// to use.
-#[derive(
-    Debug,
-    PartialEq,
-    PartialOrd,
-    Clone,
-    Copy,
-    Hash,
-    SerializeConst,
-    serde::Serialize,
-    serde::Deserialize,
-)]
+#[derive(Debug, Eq, Clone, Copy, SerializeConst, serde::Serialize, serde::Deserialize)]
 pub struct BundledAsset {
     /// The absolute path of the asset
     absolute_source_path: ConstStr,
@@ -27,13 +17,46 @@ pub struct BundledAsset {
     options: AssetOptions,
 }
 
+impl PartialEq for BundledAsset {
+    fn eq(&self, other: &Self) -> bool {
+        self.absolute_source_path == other.absolute_source_path
+            && self.bundled_path == other.bundled_path
+            && self.options == other.options
+    }
+}
+
+impl PartialOrd for BundledAsset {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        match self
+            .absolute_source_path
+            .partial_cmp(&other.absolute_source_path)
+        {
+            Some(core::cmp::Ordering::Equal) => {}
+            ord => return ord,
+        }
+        match self.bundled_path.partial_cmp(&other.bundled_path) {
+            Some(core::cmp::Ordering::Equal) => {}
+            ord => return ord,
+        }
+        self.options.partial_cmp(&other.options)
+    }
+}
+
+impl Hash for BundledAsset {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        self.absolute_source_path.hash(state);
+        self.bundled_path.hash(state);
+        self.options.hash(state);
+    }
+}
+
 impl BundledAsset {
     #[doc(hidden)]
     /// This should only be called from the macro
     /// Create a new asset
     pub const fn new(
-        absolute_source_path: &'static str,
-        bundled_path: &'static str,
+        absolute_source_path: &str,
+        bundled_path: &str,
         options: AssetOptions,
     ) -> Self {
         Self {
@@ -83,6 +106,7 @@ impl BundledAsset {
     pub fn absolute_source_path(&self) -> &str {
         self.absolute_source_path.as_str()
     }
+
     /// Get the options for the asset
     pub const fn options(&self) -> &AssetOptions {
         &self.options
@@ -101,28 +125,49 @@ impl BundledAsset {
 ///     img { src: ASSET }
 /// };
 /// ```
-#[derive(Debug, PartialEq, Clone, Copy)]
+#[derive(PartialEq, Clone, Copy)]
 pub struct Asset {
-    /// The bundled asset
-    bundled: BundledAsset,
-    /// The link section for the asset
-    keep_link_section: fn() -> u8,
+    /// A pointer to the bundled asset. This will be resolved after the linker has run and
+    /// put into the lazy asset
+    ///
+    /// WARNING: Don't read this directly. Reads can get optimized away at compile time before
+    /// the data for this is filled in by the CLI after the binary is built. Instead, use
+    /// [`std::ptr::read_volatile`] to read the data.
+    bundled: &'static [u8],
 }
 
+impl Debug for Asset {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        self.resolve().fmt(f)
+    }
+}
+
+unsafe impl Send for Asset {}
+unsafe impl Sync for Asset {}
+
 impl Asset {
     #[doc(hidden)]
     /// This should only be called from the macro
     /// Create a new asset from the bundled form of the asset and the link section
-    pub const fn new(bundled: BundledAsset, keep_link_section: fn() -> u8) -> Self {
-        Self {
-            bundled,
-            keep_link_section,
-        }
+    pub const fn new(bundled: &'static [u8]) -> Self {
+        Self { bundled }
     }
 
     /// Get the bundled asset
-    pub const fn bundled(&self) -> &BundledAsset {
-        &self.bundled
+    pub fn bundled(&self) -> BundledAsset {
+        let len = self.bundled.len();
+        let ptr = self.bundled as *const [u8] as *const u8;
+        if ptr.is_null() {
+            panic!("Tried to use an asset that was not bundled. Make sure you are compiling dx as the linker");
+        }
+        let mut bytes = ConstVec::new();
+        for byte in 0..len {
+            // SAFETY: We checked that the pointer was not null above. The pointer is valid for reads and
+            // since we are reading a u8 there are no alignment requirements
+            bytes = bytes.push(unsafe { std::ptr::read_volatile(ptr.add(byte)) });
+        }
+        let read = bytes.read();
+        deserialize_const!(BundledAsset, read).expect("Failed to deserialize asset. Make sure you built with the matching version of the Dioxus CLI").1
     }
 
     /// Return a canonicalized path to the asset
@@ -130,13 +175,10 @@ impl Asset {
     /// Attempts to resolve it against an `assets` folder in the current directory.
     /// If that doesn't exist, it will resolve against the cargo manifest dir
     pub fn resolve(&self) -> PathBuf {
-        // Force a volatile read of the asset link section to ensure the symbol makes it into the binary
-        (self.keep_link_section)();
-
         #[cfg(feature = "dioxus")]
         // If the asset is relative, we resolve the asset at the current directory
         if !dioxus_core_types::is_bundled_app() {
-            return PathBuf::from(self.bundled.absolute_source_path.as_str());
+            return PathBuf::from(self.bundled().absolute_source_path.as_str());
         }
 
         #[cfg(feature = "dioxus")]
@@ -156,7 +198,7 @@ impl Asset {
 
         // Otherwise presumably we're bundled and we can use the bundled path
         bundle_root.join(PathBuf::from(
-            self.bundled.bundled_path.as_str().trim_start_matches('/'),
+            self.bundled().bundled_path.as_str().trim_start_matches('/'),
         ))
     }
 }

packages/manganis/manganis-core/src/css.rs

@@ -5,6 +5,7 @@ use std::collections::HashSet;
 /// Options for a css asset
 #[derive(
     Debug,
+    Eq,
     PartialEq,
     PartialOrd,
     Clone,
@@ -79,6 +80,7 @@ impl CssAssetOptions {
 /// Options for a css module asset
 #[derive(
     Debug,
+    Eq,
     PartialEq,
     PartialOrd,
     Clone,

packages/manganis/manganis-core/src/folder.rs

@@ -5,6 +5,7 @@ use crate::AssetOptions;
 /// The builder for a folder asset.
 #[derive(
     Debug,
+    Eq,
     PartialEq,
     PartialOrd,
     Clone,

packages/manganis/manganis-core/src/hash.rs

@@ -1,94 +0,0 @@
-//! Utilities for creating hashed paths to assets in Manganis. This module defines [`AssetHash`] which is used to create a hashed path to an asset in both the CLI and the macro.
-
-use std::{
-    error::Error,
-    hash::{Hash, Hasher},
-    io::Read,
-    path::{Path, PathBuf},
-};
-
-/// An error that can occur when hashing an asset
-#[derive(Debug)]
-#[non_exhaustive]
-pub enum AssetHashError {
-    /// An io error occurred
-    IoError { err: std::io::Error, path: PathBuf },
-}
-
-impl std::fmt::Display for AssetHashError {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            AssetHashError::IoError { path, err } => {
-                write!(f, "Failed to read file: {}; {}", path.display(), err)
-            }
-        }
-    }
-}
-
-impl Error for AssetHashError {}
-
-/// The opaque hash type manganis uses to identify assets. Each time an asset or asset options change, this hash will
-/// change. This hash is included in the URL of the bundled asset for cache busting.
-pub struct AssetHash {
-    /// We use a wrapper type here to hide the exact size of the hash so we can switch to a sha hash in a minor version bump
-    hash: [u8; 8],
-}
-
-impl AssetHash {
-    /// Create a new asset hash
-    const fn new(hash: u64) -> Self {
-        Self {
-            hash: hash.to_le_bytes(),
-        }
-    }
-
-    /// Get the hash bytes
-    pub const fn bytes(&self) -> &[u8] {
-        &self.hash
-    }
-
-    /// Create a new asset hash for a file. The input file to this function should be fully resolved
-    pub fn hash_file_contents(file_path: &Path) -> Result<AssetHash, AssetHashError> {
-        // Create a hasher
-        let mut hash = std::collections::hash_map::DefaultHasher::new();
-
-        // If this is a folder, hash the folder contents
-        if file_path.is_dir() {
-            let files = std::fs::read_dir(file_path).map_err(|err| AssetHashError::IoError {
-                err,
-                path: file_path.to_path_buf(),
-            })?;
-            for file in files.flatten() {
-                let path = file.path();
-                Self::hash_file_contents(&path)?.bytes().hash(&mut hash);
-            }
-            let hash = hash.finish();
-            return Ok(AssetHash::new(hash));
-        }
-
-        // Otherwise, open the file to get its contents
-        let mut file = std::fs::File::open(file_path).map_err(|err| AssetHashError::IoError {
-            err,
-            path: file_path.to_path_buf(),
-        })?;
-
-        // We add a hash to the end of the file so it is invalidated when the bundled version of the file changes
-        // The hash includes the file contents, the options, and the version of manganis. From the macro, we just
-        // know the file contents, so we only include that hash
-        let mut buffer = [0; 8192];
-        loop {
-            let read = file
-                .read(&mut buffer)
-                .map_err(|err| AssetHashError::IoError {
-                    err,
-                    path: file_path.to_path_buf(),
-                })?;
-            if read == 0 {
-                break;
-            }
-            hash.write(&buffer[..read]);
-        }
-
-        Ok(AssetHash::new(hash.finish()))
-    }
-}

packages/manganis/manganis-core/src/images.rs

@@ -5,6 +5,7 @@ use crate::AssetOptions;
 /// The type of an image. You can read more about the tradeoffs between image formats [here](https://developer.mozilla.org/en-US/docs/Web/Media/Formats/Image_types)
 #[derive(
     Debug,
+    Eq,
     PartialEq,
     PartialOrd,
     Clone,
@@ -31,6 +32,7 @@ pub enum ImageFormat {
 /// The size of an image asset
 #[derive(
     Debug,
+    Eq,
     PartialEq,
     PartialOrd,
     Clone,
@@ -56,6 +58,7 @@ pub enum ImageSize {
 /// Options for an image asset
 #[derive(
     Debug,
+    Eq,
     PartialEq,
     PartialOrd,
     Clone,

packages/manganis/manganis-core/src/js.rs

@@ -5,6 +5,7 @@ use crate::AssetOptions;
 /// Options for a javascript asset
 #[derive(
     Debug,
+    Eq,
     PartialEq,
     PartialOrd,
     Clone,

packages/manganis/manganis-core/src/lib.rs

@@ -15,7 +15,3 @@ pub use js::*;
 
 mod asset;
 pub use asset::*;
-
-pub mod linker;
-
-pub mod hash;

packages/manganis/manganis-core/src/linker.rs

@@ -1,71 +0,0 @@
-//! Utilities for working with Manganis assets in the linker. This module defines [`LinkSection`] which has information about what section manganis assets are stored in on each platform.
-
-/// Information about the manganis link section for a given platform
-#[derive(Debug, Clone, Copy)]
-pub struct LinkSection {
-    /// The link section we pass to the static
-    pub link_section: &'static str,
-    /// The name of the section we find in the binary
-    pub name: &'static str,
-}
-
-impl LinkSection {
-    /// The list of link sections for all supported platforms
-    pub const ALL: &'static [&'static LinkSection] =
-        &[Self::WASM, Self::MACOS, Self::WINDOWS, Self::ILLUMOS];
-
-    /// Returns the link section used in linux, android, fuchsia, psp, freebsd, and wasm32
-    pub const WASM: &'static LinkSection = &LinkSection {
-        link_section: "manganis",
-        name: "manganis",
-    };
-
-    /// Returns the link section used in macOS, iOS, tvOS
-    pub const MACOS: &'static LinkSection = &LinkSection {
-        link_section: "__DATA,manganis,regular,no_dead_strip",
-        name: "manganis",
-    };
-
-    /// Returns the link section used in windows
-    pub const WINDOWS: &'static LinkSection = &LinkSection {
-        link_section: "mg",
-        name: "mg",
-    };
-
-    /// Returns the link section used in illumos
-    pub const ILLUMOS: &'static LinkSection = &LinkSection {
-        link_section: "set_manganis",
-        name: "set_manganis",
-    };
-
-    /// The link section used on the current platform
-    pub const CURRENT: &'static LinkSection = {
-        #[cfg(any(
-            target_os = "none",
-            target_os = "linux",
-            target_os = "android",
-            target_os = "fuchsia",
-            target_os = "psp",
-            target_os = "freebsd",
-            target_arch = "wasm32"
-        ))]
-        {
-            Self::WASM
-        }
-
-        #[cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos"))]
-        {
-            Self::MACOS
-        }
-
-        #[cfg(target_os = "windows")]
-        {
-            Self::WINDOWS
-        }
-
-        #[cfg(target_os = "illumos")]
-        {
-            Self::ILLUMOS
-        }
-    };
-}

packages/manganis/manganis-core/src/options.rs

@@ -7,6 +7,7 @@ use crate::{
 /// Settings for a generic asset
 #[derive(
     Debug,
+    Eq,
     PartialEq,
     PartialOrd,
     Clone,