Restore manganis optimizations (#3195)

* Create a crate for constant serialization of config structs for manganis
* use SerializeConst for the image asset builder
* switch to a declarative macro for assets
* add serializable options for each asset type
* implement the new manganis macro
* optimize assets in the CLI
* Fix assets with dioxus formattable
* reduce fuzzing test limits
* use the new syntax in examples
* fix avif support
* Fix manganis doc tests
* cache asset optimization
* Split out asset and bundled asset
* make hash pointer width independent
* Fix manganis macro docs
* add constvec::is_empty method to fix clippy lint
* remove nasm feature
* simplify test_rsplit_once test so typos passes
* revert example change from clippy fix
* always accept unix paths
* fix windows path seperator
* fix folder asset hash
* Optimize assets in a blocking task
* Fix asset options docs
* Document Asset and BundledAsset
* simplify the linker macro a bit
* add more docs to AssetParser expansion
* add image format helpers
* Split by  both unix and windows path separators and take the smallest one
This commit is contained in:
Evan Almloff authored and GitHub committed 2024-11-26 18:12:24 +01:00
1 parent 31d72db82d
commit f9617c8d33
64 files changed
+6233 -704

No files matched your search

+150 -112
View File
@@ -1,118 +1,156 @@
use serde::{Deserialize, Serialize};
use std::{
hash::{Hash, Hasher},
path::PathBuf,
time::SystemTime,
};
use crate::AssetOptions;
use const_serialize::{ConstStr, SerializeConst};
use std::path::PathBuf;
/// The location we'll write to the link section - needs to be serializable
///
/// This basically is 1:1 with `manganis/Asset` but with more metadata to be useful to the macro and cli
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)]
pub struct ResourceAsset {
/// The input path `/assets/blah.css`
pub input: PathBuf,
/// The canonicalized asset
///
/// `Users/dioxus/dev/app/assets/blah.css`
pub absolute: PathBuf,
/// The post-bundle name of the asset - do we include the `assets` name?
///
/// `blahcss123.css`
pub bundled: String,
/// An asset that should be copied by the bundler with some options. This type will be
/// serialized into the binary and added to the link section [`LinkSection::CURRENT`](crate::linker::LinkSection::CURRENT).
/// CLIs that support manganis, should pull out the assets from the link section, optimize,
/// and write them to the filesystem at [`BundledAsset::bundled_path`] for the application
/// to use.
#[derive(
Debug,
PartialEq,
PartialOrd,
Clone,
Copy,
Hash,
SerializeConst,
serde::Serialize,
serde::Deserialize,
)]
pub struct BundledAsset {
/// The absolute path of the asset
absolute_source_path: ConstStr,
/// The bundled path of the asset
bundled_path: ConstStr,
/// The options for the asset
options: AssetOptions,
}
impl ResourceAsset {
pub fn parse_any(raw: &str) -> Result<Self, AssetParseError> {
// get the location where the asset is absolute, relative to
//
// IE
// /users/dioxus/dev/app/
// is the root of
// /users/dioxus/dev/app/assets/blah.css
let manifest_dir = std::env::var("CARGO_MANIFEST_DIR")
.map(PathBuf::from)
.unwrap();
// 1. the input file should be a pathbuf
let input = PathBuf::from(raw);
// 2. absolute path to the asset
let absolute = manifest_dir.join(raw.trim_start_matches('/'));
let absolute =
absolute
.canonicalize()
.map_err(|err| AssetParseError::AssetDoesntExist {
err,
path: absolute,
})?;
// 3. the bundled path is the unique name of the asset
let bundled = Self::make_unique_name(absolute.clone())?;
Ok(Self {
input,
absolute,
bundled,
})
}
fn make_unique_name(file_path: PathBuf) -> Result<String, AssetParseError> {
// Create a hasher
let mut hash = std::collections::hash_map::DefaultHasher::new();
// Open the file to get its options
let file = std::fs::File::open(&file_path).map_err(AssetParseError::FailedToReadAsset)?;
let modified = file
.metadata()
.and_then(|metadata| metadata.modified())
.unwrap_or(SystemTime::UNIX_EPOCH);
// Hash a bunch of metadata
// name, options, modified time, and maybe the version of our crate
// Hash the last time the file was updated and the file source. If either of these change, we need to regenerate the unique name
modified.hash(&mut hash);
file_path.hash(&mut hash);
let uuid = hash.finish();
let file_name = file_path
.file_stem()
.expect("file_path should have a file_stem")
.to_string_lossy();
let extension = file_path
.extension()
.map(|f| f.to_string_lossy())
.unwrap_or_default();
Ok(format!("{file_name}-{uuid:x}.{extension}"))
}
}
#[derive(Debug)]
pub enum AssetParseError {
ParseError(String),
AssetDoesntExist {
err: std::io::Error,
path: std::path::PathBuf,
},
FailedToReadAsset(std::io::Error),
FailedToReadMetadata(std::io::Error),
}
impl std::fmt::Display for AssetParseError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
AssetParseError::ParseError(err) => write!(f, "Failed to parse asset: {}", err),
AssetParseError::AssetDoesntExist { err, path } => {
write!(f, "Asset at {} doesn't exist: {}", path.display(), err)
}
AssetParseError::FailedToReadAsset(err) => write!(f, "Failed to read asset: {}", err),
AssetParseError::FailedToReadMetadata(err) => {
write!(f, "Failed to read asset metadata: {}", err)
}
impl BundledAsset {
#[doc(hidden)]
/// This should only be called from the macro
/// Create a new asset
pub const fn new(
absolute_source_path: &'static str,
bundled_path: &'static str,
options: AssetOptions,
) -> Self {
Self {
absolute_source_path: ConstStr::new(absolute_source_path),
bundled_path: ConstStr::new(bundled_path),
options,
}
}
#[doc(hidden)]
/// This should only be called from the macro
/// Create a new asset but with a relative path
///
/// This method is deprecated and will be removed in a future release.
#[deprecated(
note = "Relative asset!() paths are not supported. Use a path like `/assets/myfile.png` instead of `./assets/myfile.png`"
)]
pub const fn new_relative(
absolute_source_path: &'static str,
bundled_path: &'static str,
options: AssetOptions,
) -> Self {
Self::new(absolute_source_path, bundled_path, options)
}
/// Get the bundled name of the asset. This identifier cannot be used to read the asset directly
pub fn bundled_path(&self) -> &str {
self.bundled_path.as_str()
}
/// Get the absolute path of the asset source. This path will not be available when the asset is bundled
pub fn absolute_source_path(&self) -> &str {
self.absolute_source_path.as_str()
}
/// Get the options for the asset
pub const fn options(&self) -> &AssetOptions {
&self.options
}
}
/// A bundled asset with some options. The asset can be used in rsx! to reference the asset.
/// It should not be read directly with [`std::fs::read`] because the path needs to be resolved
/// relative to the bundle
///
/// ```rust
/// # use manganis::{asset, Asset};
/// # use dioxus::prelude::*;
/// const ASSET: Asset = asset!("/assets/image.png");
/// rsx! {
/// img { src: ASSET }
/// };
/// ```
#[derive(Debug, PartialEq, Clone, Copy)]
pub struct Asset {
/// The bundled asset
bundled: BundledAsset,
/// The link section for the asset
keep_link_section: fn() -> u8,
}
impl Asset {
#[doc(hidden)]
/// This should only be called from the macro
/// Create a new asset from the bundled form of the asset and the link section
pub const fn new(bundled: BundledAsset, keep_link_section: fn() -> u8) -> Self {
Self {
bundled,
keep_link_section,
}
}
/// Get the bundled asset
pub const fn bundled(&self) -> &BundledAsset {
&self.bundled
}
/// Return a canonicalized path to the asset
///
/// Attempts to resolve it against an `assets` folder in the current directory.
/// If that doesn't exist, it will resolve against the cargo manifest dir
pub fn resolve(&self) -> PathBuf {
// Force a volatile read of the asset link section to ensure the symbol makes it into the binary
(self.keep_link_section)();
#[cfg(feature = "dioxus")]
// If the asset is relative, we resolve the asset at the current directory
if !dioxus_core_types::is_bundled_app() {
return PathBuf::from(self.bundled.absolute_source_path.as_str());
}
// Otherwise presumably we're bundled and we can use the bundled path
PathBuf::from("/assets/").join(PathBuf::from(
self.bundled.bundled_path.as_str().trim_start_matches('/'),
))
}
}
impl From<Asset> for String {
fn from(value: Asset) -> Self {
value.to_string()
}
}
impl From<Asset> for Option<String> {
fn from(value: Asset) -> Self {
Some(value.to_string())
}
}
impl std::fmt::Display for Asset {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.resolve().display())
}
}
#[cfg(feature = "dioxus")]
impl dioxus_core_types::DioxusFormattable for Asset {
fn format(&self) -> std::borrow::Cow<'static, str> {
std::borrow::Cow::Owned(self.to_string())
}
}
@@ -0,0 +1,77 @@
use const_serialize::SerializeConst;
use crate::AssetOptions;
/// Options for a css asset
#[derive(
Debug,
PartialEq,
PartialOrd,
Clone,
Copy,
Hash,
SerializeConst,
serde::Serialize,
serde::Deserialize,
)]
pub struct CssAssetOptions {
minify: bool,
preload: bool,
}
impl Default for CssAssetOptions {
fn default() -> Self {
Self::new()
}
}
impl CssAssetOptions {
/// Create a new css asset using the builder
pub const fn new() -> Self {
Self {
preload: false,
minify: true,
}
}
/// Sets whether the css should be minified (default: true)
///
/// Minifying the css can make your site load faster by loading less data
///
/// ```rust
/// # use manganis::{asset, Asset, CssAssetOptions};
/// const _: Asset = asset!("/assets/style.css", CssAssetOptions::new().with_minify(false));
/// ```
#[allow(unused)]
pub const fn with_minify(self, minify: bool) -> Self {
Self { minify, ..self }
}
/// Check if the asset is minified
pub const fn minified(&self) -> bool {
self.minify
}
/// Make the asset preloaded
///
/// Preloading css will make the image start to load as soon as possible. This is useful for css that is used soon after the page loads or css that may not be used immediately, but should start loading sooner
///
/// ```rust
/// # use manganis::{asset, Asset, CssAssetOptions};
/// const _: Asset = asset!("/assets/style.css", CssAssetOptions::new().with_preload(true));
/// ```
#[allow(unused)]
pub const fn with_preload(self, preload: bool) -> Self {
Self { preload, ..self }
}
/// Check if the asset is preloaded
pub const fn preloaded(&self) -> bool {
self.preload
}
/// Convert the options into options for a generic asset
pub const fn into_asset_options(self) -> AssetOptions {
AssetOptions::Css(self)
}
}
@@ -0,0 +1,35 @@
use const_serialize::SerializeConst;
use crate::AssetOptions;
/// The builder for [`FolderAsset`]
#[derive(
Debug,
PartialEq,
PartialOrd,
Clone,
Copy,
Hash,
SerializeConst,
serde::Serialize,
serde::Deserialize,
)]
pub struct FolderAssetOptions {}
impl Default for FolderAssetOptions {
fn default() -> Self {
Self::new()
}
}
impl FolderAssetOptions {
/// Create a new folder asset using the builder
pub const fn new() -> Self {
Self {}
}
/// Convert the options into options for a generic asset
pub const fn into_asset_options(self) -> AssetOptions {
AssetOptions::Folder(self)
}
}
@@ -0,0 +1,225 @@
use const_serialize::SerializeConst;
use crate::AssetOptions;
/// The type of an image. You can read more about the tradeoffs between image formats [here](https://developer.mozilla.org/en-US/docs/Web/Media/Formats/Image_types)
#[derive(
Debug,
PartialEq,
PartialOrd,
Clone,
Copy,
Hash,
SerializeConst,
serde::Serialize,
serde::Deserialize,
)]
#[repr(u8)]
pub enum ImageFormat {
/// A png image. Png images cannot contain transparency and tend to compress worse than other formats
Png,
/// A jpg image. Jpg images can contain transparency and tend to compress better than png images
Jpg,
/// A webp image. Webp images can contain transparency and tend to compress better than jpg images
Webp,
/// An avif image. Avif images can compress slightly better than webp images but are not supported by all browsers
Avif,
/// An unknown image type
Unknown,
}
/// The size of an image asset
#[derive(
Debug,
PartialEq,
PartialOrd,
Clone,
Copy,
Hash,
SerializeConst,
serde::Serialize,
serde::Deserialize,
)]
#[repr(C, u8)]
pub enum ImageSize {
/// A manual size in pixels
Manual {
/// The width of the image in pixels
width: u32,
/// The height of the image in pixels
height: u32,
},
/// The size will be automatically determined from the image source
Automatic,
}
/// Options for an image asset
#[derive(
Debug,
PartialEq,
PartialOrd,
Clone,
Copy,
Hash,
SerializeConst,
serde::Serialize,
serde::Deserialize,
)]
pub struct ImageAssetOptions {
ty: ImageFormat,
low_quality_preview: bool,
size: ImageSize,
preload: bool,
}
impl Default for ImageAssetOptions {
fn default() -> Self {
Self::new()
}
}
impl ImageAssetOptions {
/// Create a new image asset options
pub const fn new() -> Self {
Self {
ty: ImageFormat::Unknown,
low_quality_preview: false,
size: ImageSize::Automatic,
preload: false,
}
}
/// Make the asset preloaded
///
/// Preloading an image will make the image start to load as soon as possible. This is useful for images that will be displayed soon after the page loads or images that may not be visible immediately, but should start loading sooner
///
/// ```rust
/// # use manganis::{asset, Asset, ImageAssetOptions};
/// const _: Asset = asset!("/assets/image.png", ImageAssetOptions::new().with_preload(true));
/// ```
pub const fn with_preload(self, preload: bool) -> Self {
Self { preload, ..self }
}
/// Check if the asset is preloaded
pub const fn preloaded(&self) -> bool {
self.preload
}
/// Sets the format of the image
///
/// Choosing the right format can make your site load much faster. Webp and avif images tend to be a good default for most images
///
/// ```rust
/// # use manganis::{asset, Asset, ImageAssetOptions, ImageFormat};
/// const _: Asset = asset!("/assets/image.png", ImageAssetOptions::new().with_format(ImageFormat::Webp));
/// ```
pub const fn with_format(self, format: ImageFormat) -> Self {
Self { ty: format, ..self }
}
/// Sets the format of the image to [`ImageFormat::Avif`]
///
/// Avif images tend to be a good default for most images rendered in browser because
/// they compress images well
///
/// ```rust
/// # use manganis::{asset, Asset, ImageAssetOptions, ImageFormat};
/// const _: Asset = asset!("/assets/image.png", ImageAssetOptions::new().with_avif());
/// ```
pub const fn with_avif(self) -> Self {
self.with_format(ImageFormat::Avif)
}
/// Sets the format of the image to [`ImageFormat::Webp`]
///
/// Webp images tend to be a good default for most images rendered in browser because
/// they compress images well
///
/// ```rust
/// # use manganis::{asset, Asset, ImageAssetOptions, ImageFormat};
/// const _: Asset = asset!("/assets/image.png", ImageAssetOptions::new().with_webp());
/// ```
pub const fn with_webp(self) -> Self {
self.with_format(ImageFormat::Webp)
}
/// Sets the format of the image to [`ImageFormat::Jpg`]
///
/// Jpeg images compress much better than [`ImageFormat::Png`], but worse than [`ImageFormat::Webp`] or [`ImageFormat::Avif`]
///
/// ```rust
/// # use manganis::{asset, Asset, ImageAssetOptions, ImageFormat};
/// const _: Asset = asset!("/assets/image.png", ImageAssetOptions::new().with_jpg());
/// ```
pub const fn with_jpg(self) -> Self {
self.with_format(ImageFormat::Jpg)
}
/// Sets the format of the image to [`ImageFormat::Png`]
///
/// Png images don't compress very well, so they are not recommended for large images
///
/// ```rust
/// # use manganis::{asset, Asset, ImageAssetOptions, ImageFormat};
/// const _: Asset = asset!("/assets/image.png", ImageAssetOptions::new().with_png());
/// ```
pub const fn with_png(self) -> Self {
self.with_format(ImageFormat::Png)
}
/// Get the format of the image
pub const fn format(&self) -> ImageFormat {
self.ty
}
/// Sets the size of the image
///
/// If you only use the image in one place, you can set the size of the image to the size it will be displayed at. This will make the image load faster
///
/// ```rust
/// # use manganis::{asset, Asset, ImageAssetOptions, ImageSize};
/// const _: Asset = asset!("/assets/image.png", ImageAssetOptions::new().with_size(ImageSize::Manual { width: 512, height: 512 }));
/// ```
pub const fn with_size(self, size: ImageSize) -> Self {
Self { size, ..self }
}
/// Get the size of the image
pub const fn size(&self) -> ImageSize {
self.size
}
// LQIP is currently disabled until we have the CLI set up to inject the low quality image preview after the crate is built through the linker
// /// Make the image use a low quality preview
// ///
// /// A low quality preview is a small version of the image that will load faster. This is useful for large images on mobile devices that may take longer to load
// ///
// /// ```rust
// /// # use manganis::{asset, Asset, ImageAssetOptions};
// /// const _: Asset = manganis::asset!("/assets/image.png", ImageAssetOptions::new().with_low_quality_image_preview());
// /// ```
//
// pub const fn with_low_quality_image_preview(self, low_quality_preview: bool) -> Self {
// Self {
// low_quality_preview,
// ..self
// }
// }
/// Convert the options into options for a generic asset
pub const fn into_asset_options(self) -> AssetOptions {
AssetOptions::Image(self)
}
pub(crate) const fn extension(&self) -> Option<&'static str> {
match self.ty {
ImageFormat::Png => Some("png"),
ImageFormat::Jpg => Some("jpg"),
ImageFormat::Webp => Some("webp"),
ImageFormat::Avif => Some("avif"),
ImageFormat::Unknown => None,
}
}
}
+77
View File
@@ -0,0 +1,77 @@
use const_serialize::SerializeConst;
use crate::AssetOptions;
/// Options for a javascript asset
#[derive(
Debug,
PartialEq,
PartialOrd,
Clone,
Copy,
Hash,
SerializeConst,
serde::Serialize,
serde::Deserialize,
)]
pub struct JsAssetOptions {
minify: bool,
preload: bool,
}
impl Default for JsAssetOptions {
fn default() -> Self {
Self::new()
}
}
impl JsAssetOptions {
/// Create a new js asset builder
pub const fn new() -> Self {
Self {
minify: true,
preload: false,
}
}
/// Sets whether the js should be minified (default: true)
///
/// Minifying the js can make your site load faster by loading less data
///
/// ```rust
/// # use manganis::{asset, Asset, JsAssetOptions};
/// const _: Asset = asset!("/assets/script.js", JsAssetOptions::new().with_minify(false));
/// ```
#[allow(unused)]
pub const fn with_minify(self, minify: bool) -> Self {
Self { minify, ..self }
}
/// Check if the asset is minified
pub const fn minified(&self) -> bool {
self.minify
}
/// Make the asset preloaded
///
/// Preloading the javascript will make the javascript start to load as soon as possible. This is useful for javascript that will be used soon after the page loads or javascript that may not be used immediately, but should start loading sooner
///
/// ```rust
/// # use manganis::{asset, Asset, JsAssetOptions};
/// const _: Asset = asset!("/assets/script.js", JsAssetOptions::new().with_preload(true));
/// ```
#[allow(unused)]
pub const fn with_preload(self, preload: bool) -> Self {
Self { preload, ..self }
}
/// Check if the asset is preloaded
pub const fn preloaded(&self) -> bool {
self.preload
}
/// Convert the options into options for a generic asset
pub const fn into_asset_options(self) -> AssetOptions {
AssetOptions::Js(self)
}
}
+17 -3
View File
@@ -1,5 +1,19 @@
mod asset;
mod linker;
mod folder;
pub use folder::*;
mod images;
pub use images::*;
mod options;
pub use options::*;
mod css;
pub use css::*;
mod js;
pub use js::*;
mod asset;
pub use asset::*;
pub use linker::*;
pub mod linker;
@@ -1,3 +1,5 @@
//! Utilities for working with Manganis assets in the linker. This module defines [`LinkSection`] which has information about what section manganis assets are stored in on each platform.
/// Information about the manganis link section for a given platform
#[derive(Debug, Clone, Copy)]
pub struct LinkSection {
@@ -0,0 +1,48 @@
use const_serialize::SerializeConst;
use crate::{CssAssetOptions, FolderAssetOptions, ImageAssetOptions, JsAssetOptions};
/// Settings for a generic asset
#[derive(
Debug,
PartialEq,
PartialOrd,
Clone,
Copy,
Hash,
SerializeConst,
serde::Serialize,
serde::Deserialize,
)]
#[repr(C, u8)]
#[non_exhaustive]
pub enum AssetOptions {
/// An image asset
Image(ImageAssetOptions),
/// A folder asset
Folder(FolderAssetOptions),
/// A css asset
Css(CssAssetOptions),
/// A javascript asset
Js(JsAssetOptions),
/// An unknown asset
Unknown,
}
impl AssetOptions {
/// Try to get the extension for the asset. If the asset options don't define an extension, this will return None
pub const fn extension(&self) -> Option<&'static str> {
match self {
AssetOptions::Image(image) => image.extension(),
AssetOptions::Css(_) => Some("css"),
AssetOptions::Js(_) => Some("js"),
AssetOptions::Folder(_) => None,
AssetOptions::Unknown => None,
}
}
/// Convert the options into options for a generic asset
pub const fn into_asset_options(self) -> Self {
self
}
}