Restore manganis optimizations (#3195)
* Create a crate for constant serialization of config structs for manganis * use SerializeConst for the image asset builder * switch to a declarative macro for assets * add serializable options for each asset type * implement the new manganis macro * optimize assets in the CLI * Fix assets with dioxus formattable * reduce fuzzing test limits * use the new syntax in examples * fix avif support * Fix manganis doc tests * cache asset optimization * Split out asset and bundled asset * make hash pointer width independent * Fix manganis macro docs * add constvec::is_empty method to fix clippy lint * remove nasm feature * simplify test_rsplit_once test so typos passes * revert example change from clippy fix * always accept unix paths * fix windows path seperator * fix folder asset hash * Optimize assets in a blocking task * Fix asset options docs * Document Asset and BundledAsset * simplify the linker macro a bit * add more docs to AssetParser expansion * add image format helpers * Split by both unix and windows path separators and take the smallest one
This commit is contained in:
Evan Almloff authored and GitHub committed
1 parent
31d72db82d
commit
f9617c8d33
64 files changed
+6233
-704
No files matched your search
@@ -13,29 +13,10 @@ keywords = ["assets"]
|
||||
[lib]
|
||||
|
||||
[dependencies]
|
||||
manganis-macro = { workspace = true, optional = true }
|
||||
dioxus-core-types = { workspace = true }
|
||||
|
||||
# dunce = "1.0.2"
|
||||
# [target.'cfg(any(target_os = "macos", target_os = "ios"))'.dependencies]
|
||||
# core-foundation = "0.10.0"
|
||||
# [target.'cfg(target_os = "macos")'.dependencies]
|
||||
# core-foundation = "0.9.3"
|
||||
# [target.'cfg(target_os ="macos")'.dependencies]
|
||||
# infer = { workspace = true }
|
||||
# dirs = "5.0.1"
|
||||
# infer = { workspace = true }
|
||||
# manganis-common = { workspace = true }
|
||||
|
||||
once_cell = "1.19.0"
|
||||
dunce = "1.0.2"
|
||||
serde = { version = "1.0.183", features = ["derive"] }
|
||||
anyhow = "1"
|
||||
base64 = { workspace = true }
|
||||
|
||||
const-serialize = { workspace = true }
|
||||
manganis-core = { workspace = true }
|
||||
manganis-macro = { workspace = true }
|
||||
|
||||
[features]
|
||||
default = ["macro"]
|
||||
html = []
|
||||
# url-encoding = ["manganis-macro/url-encoding"]
|
||||
macro = ["dep:manganis-macro"]
|
||||
default = []
|
||||
dioxus = ["manganis-core/dioxus"]
|
||||
@@ -0,0 +1,31 @@
|
||||
# Manganis
|
||||
|
||||
The Manganis allows you to submit assets to any build tool that supports collecting assets. It makes it easy to self-host assets that are distributed throughout your libraries. Manganis also handles optimizing, converting, and fetching assets.
|
||||
|
||||
If you defined this in a component library:
|
||||
|
||||
```rust
|
||||
use manganis::{Asset, asset};
|
||||
const AVIF_ASSET: Asset = manganis::asset!("/assets/image.png");
|
||||
```
|
||||
|
||||
AVIF_ASSET will be set to a new file name that will be served by some CLI. That file can be collected by any package that depends on the component library.
|
||||
|
||||
```rust
|
||||
use manganis::{ImageFormat, ImageAssetOptions, Asset, asset, ImageSize};
|
||||
// You can collect arbitrary files. Absolute paths are resolved relative to the package root
|
||||
const _: Asset = asset!("/assets/script.js");
|
||||
|
||||
// You can collect images which will be automatically optimized
|
||||
pub const PNG_ASSET: Asset =
|
||||
asset!("/assets/image.png");
|
||||
// Resize the image at compile time to make the assets smaller
|
||||
pub const RESIZED_PNG_ASSET: Asset =
|
||||
asset!("/assets/image.png", ImageAssetOptions::new().with_size(ImageSize::Manual { width: 52, height: 52 }));
|
||||
// Or convert the image at compile time to a web friendly format
|
||||
pub const AVIF_ASSET: Asset = asset!("/assets/image.png", ImageAssetOptions::new().with_format(ImageFormat::Avif));
|
||||
```
|
||||
|
||||
## Adding Support to Your CLI
|
||||
|
||||
To add support for your CLI, you need to integrate with the [manganis_cli_support](https://github.com/DioxusLabs/manganis/tree/main/cli-support) crate. This crate provides utilities to collect assets that integrate with the Manganis macro. It makes it easy to integrate an asset collection and optimization system into a build tool.
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 3.2 KiB |
Whitespace-only changes.
Whitespace-only changes.
@@ -1,87 +0,0 @@
|
||||
use dioxus_core_types::DioxusFormattable;
|
||||
use std::path::PathBuf;
|
||||
|
||||
/// Asset
|
||||
#[derive(Debug, PartialEq, PartialOrd, Clone, Copy, Hash)]
|
||||
pub struct Asset {
|
||||
/// The input URI given to the macro
|
||||
pub input: &'static str,
|
||||
|
||||
/// The absolute path to the asset on the filesystem
|
||||
pub local: &'static str,
|
||||
|
||||
/// The asset location after its been bundled
|
||||
///
|
||||
/// `blah-123.css``
|
||||
pub bundled: &'static str,
|
||||
|
||||
/// The metadata for this asset plumbed by the compiler
|
||||
pub metadata: fn() -> u8,
|
||||
}
|
||||
|
||||
impl Asset {
|
||||
/// Create a new asset
|
||||
pub const fn new(self) -> Self {
|
||||
self
|
||||
}
|
||||
|
||||
/// Create a new asset but with a relative path
|
||||
///
|
||||
/// This method is deprecated and will be removed in a future release.
|
||||
#[deprecated(
|
||||
note = "Relative asset!() paths are not supported. Use a path like `/assets/myfile.png` instead of `./assets/myfile.png`"
|
||||
)]
|
||||
pub const fn new_relative(self) -> Self {
|
||||
self
|
||||
}
|
||||
|
||||
/// Get the path to the asset
|
||||
pub fn path(&self) -> PathBuf {
|
||||
PathBuf::from(self.input.to_string())
|
||||
}
|
||||
|
||||
/// Get the path to the asset
|
||||
pub fn relative_path(&self) -> PathBuf {
|
||||
PathBuf::from(self.input.trim_start_matches('/').to_string())
|
||||
}
|
||||
|
||||
/// Return a canonicalized path to the asset
|
||||
///
|
||||
/// Attempts to resolve it against an `assets` folder in the current directory.
|
||||
/// If that doesn't exist, it will resolve against the cargo manifest dir
|
||||
pub fn resolve(&self) -> PathBuf {
|
||||
// Force a volatile read of the metadata to ensure the symbol makes it into the binary
|
||||
(self.metadata)();
|
||||
|
||||
// If the asset is relative, we resolve the asset at the current directory
|
||||
if !dioxus_core_types::is_bundled_app() {
|
||||
return PathBuf::from(self.local);
|
||||
}
|
||||
|
||||
// Otherwise presumably we're bundled and we can use the bundled path
|
||||
PathBuf::from("/assets/").join(PathBuf::from(self.bundled.trim_start_matches('/')))
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Asset> for String {
|
||||
fn from(value: Asset) -> Self {
|
||||
value.to_string()
|
||||
}
|
||||
}
|
||||
impl From<Asset> for Option<String> {
|
||||
fn from(value: Asset) -> Self {
|
||||
Some(value.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for Asset {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{}", self.resolve().display())
|
||||
}
|
||||
}
|
||||
|
||||
impl DioxusFormattable for Asset {
|
||||
fn format(&self) -> std::borrow::Cow<'static, str> {
|
||||
std::borrow::Cow::Owned(self.to_string())
|
||||
}
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
use crate::Asset;
|
||||
|
||||
/// This is basically a compile-time version of ResourceAsset
|
||||
/// A struct that contains the relative and absolute paths of an asset
|
||||
#[derive(Debug, PartialEq, PartialOrd, Clone, Hash)]
|
||||
pub struct FolderAsset {
|
||||
src: Asset,
|
||||
}
|
||||
|
||||
impl Asset {
|
||||
/// Create a new folder using `Asset` as the base type
|
||||
pub const fn folder(self) -> FolderAsset {
|
||||
FolderAsset::new(self)
|
||||
}
|
||||
}
|
||||
|
||||
impl FolderAsset {
|
||||
/// Create a new folder asset from an `Asset`
|
||||
pub const fn new(src: Asset) -> Self {
|
||||
Self { src }
|
||||
}
|
||||
}
|
||||
|
||||
/// The builder for [`FolderAsset`]
|
||||
pub struct FolderAssetBuilder;
|
||||
|
||||
/// Create an folder asset from the local path
|
||||
#[allow(unused)]
|
||||
pub const fn folder(src: &'static str) -> FolderAssetBuilder {
|
||||
FolderAssetBuilder {}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
use const_serialize::{serialize_const, ConstVec, SerializeConst};
|
||||
|
||||
// From rustchash - https://github.com/rust-lang/rustc-hash/blob/6745258da00b7251bed4a8461871522d0231a9c7/src/lib.rs#L98
|
||||
const K: u64 = 0xf1357aea2e62a9c5;
|
||||
|
||||
pub(crate) struct ConstHasher {
|
||||
hash: u64,
|
||||
}
|
||||
|
||||
impl ConstHasher {
|
||||
pub const fn new() -> Self {
|
||||
Self { hash: 0 }
|
||||
}
|
||||
|
||||
pub const fn finish(&self) -> u64 {
|
||||
self.hash
|
||||
}
|
||||
|
||||
pub const fn write(mut self, bytes: &[u8]) -> Self {
|
||||
let mut i = 0;
|
||||
while i < bytes.len() {
|
||||
self = self.write_byte(bytes[i]);
|
||||
i += 1;
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
pub const fn write_byte(mut self, byte: u8) -> Self {
|
||||
self.hash = self.hash.wrapping_add(byte as u64).wrapping_mul(K);
|
||||
self
|
||||
}
|
||||
|
||||
pub const fn hash_by_bytes<T: SerializeConst>(self, item: &T) -> Self {
|
||||
let mut bytes = ConstVec::new();
|
||||
bytes = serialize_const(item, bytes);
|
||||
let bytes = bytes.as_ref();
|
||||
self.write(bytes)
|
||||
}
|
||||
}
|
||||
@@ -1,141 +0,0 @@
|
||||
use dioxus_core_types::DioxusFormattable;
|
||||
|
||||
use crate::Asset;
|
||||
|
||||
/// An image asset that is built by the [`asset!`] macro
|
||||
#[derive(Debug, PartialEq, PartialOrd, Clone, Hash, Copy)]
|
||||
pub struct ImageAsset {
|
||||
/// The path to the image
|
||||
asset: Asset,
|
||||
/// A low quality preview of the image that is URL encoded
|
||||
preview: Option<&'static str>,
|
||||
/// A caption for the image
|
||||
caption: Option<&'static str>,
|
||||
/// The format of the image
|
||||
format: Option<ImageType>,
|
||||
}
|
||||
|
||||
impl Asset {
|
||||
/// Convert this asset into an image asset
|
||||
pub const fn image(self) -> ImageAsset {
|
||||
ImageAsset::new(self)
|
||||
}
|
||||
}
|
||||
|
||||
impl ImageAsset {
|
||||
/// Creates a new image asset
|
||||
pub const fn new(path: Asset) -> Self {
|
||||
Self {
|
||||
asset: path,
|
||||
preview: None,
|
||||
caption: None,
|
||||
format: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the path to the image
|
||||
pub const fn path(&self) -> &'static str {
|
||||
self.asset.bundled
|
||||
}
|
||||
|
||||
/// Returns the preview of the image
|
||||
pub const fn preview(&self) -> Option<&'static str> {
|
||||
self.preview
|
||||
}
|
||||
|
||||
/// Sets the preview of the image
|
||||
pub const fn with_preview(self, preview: Option<&'static str>) -> Self {
|
||||
Self { preview, ..self }
|
||||
}
|
||||
|
||||
/// Returns the caption of the image
|
||||
pub const fn caption(&self) -> Option<&'static str> {
|
||||
self.caption
|
||||
}
|
||||
|
||||
/// Sets the caption of the image
|
||||
pub const fn with_caption(self, caption: Option<&'static str>) -> Self {
|
||||
Self { caption, ..self }
|
||||
}
|
||||
|
||||
/// Sets the format of the image
|
||||
pub const fn format(self, format: Option<ImageType>) -> Self {
|
||||
Self { format, ..self }
|
||||
}
|
||||
}
|
||||
|
||||
impl std::ops::Deref for ImageAsset {
|
||||
type Target = Asset;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.asset
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ImageAsset {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
self.asset.fmt(f)
|
||||
}
|
||||
}
|
||||
|
||||
impl DioxusFormattable for ImageAsset {
|
||||
fn format(&self) -> std::borrow::Cow<'static, str> {
|
||||
std::borrow::Cow::Owned(self.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// The type of an image. You can read more about the tradeoffs between image formats [here](https://developer.mozilla.org/en-US/docs/Web/Media/Formats/Image_types)
|
||||
#[derive(Debug, PartialEq, PartialOrd, Clone, Copy, Hash)]
|
||||
pub enum ImageType {
|
||||
/// A png image. Png images cannot contain transparency and tend to compress worse than other formats
|
||||
Png,
|
||||
/// A jpg image. Jpg images can contain transparency and tend to compress better than png images
|
||||
Jpg,
|
||||
/// A webp image. Webp images can contain transparency and tend to compress better than jpg images
|
||||
Webp,
|
||||
/// An avif image. Avif images can compress slightly better than webp images but are not supported by all browsers
|
||||
Avif,
|
||||
}
|
||||
|
||||
/// A builder for an image asset. This must be used in the [`asset!`] macro.
|
||||
///
|
||||
/// > **Note**: This will do nothing outside of the `asset!` macro
|
||||
pub struct ImageAssetBuilder;
|
||||
|
||||
impl ImageAssetBuilder {
|
||||
/// Sets the format of the image
|
||||
#[allow(unused)]
|
||||
pub const fn format(self, format: ImageType) -> Self {
|
||||
Self
|
||||
}
|
||||
|
||||
/// Sets the size of the image
|
||||
#[allow(unused)]
|
||||
pub const fn size(self, x: u32, y: u32) -> Self {
|
||||
Self
|
||||
}
|
||||
|
||||
/// Make the image use a low quality preview
|
||||
#[allow(unused)]
|
||||
pub const fn low_quality_preview(self) -> Self {
|
||||
Self
|
||||
}
|
||||
|
||||
/// Make the image preloaded
|
||||
#[allow(unused)]
|
||||
pub const fn preload(self) -> Self {
|
||||
Self
|
||||
}
|
||||
|
||||
/// Make the image URL encoded
|
||||
#[allow(unused)]
|
||||
pub const fn url_encoded(self) -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
/// Create an image asset from the local path or url to the image
|
||||
#[allow(unused)]
|
||||
pub const fn image(path: &'static str) -> ImageAssetBuilder {
|
||||
ImageAssetBuilder
|
||||
}
|
||||
@@ -1,15 +1,12 @@
|
||||
//! Manganis is a Rust library for bundling assets into a final binary.
|
||||
|
||||
#![doc = include_str!("../README.md")]
|
||||
#![deny(missing_docs)]
|
||||
|
||||
#[cfg(feature = "macro")]
|
||||
pub use manganis_macro::*;
|
||||
mod hash;
|
||||
#[doc(hidden)]
|
||||
pub mod macro_helpers;
|
||||
pub use manganis_macro::asset;
|
||||
|
||||
mod folder;
|
||||
pub use folder::*;
|
||||
|
||||
mod images;
|
||||
pub use images::*;
|
||||
|
||||
mod builder;
|
||||
pub use builder::*;
|
||||
pub use manganis_core::{
|
||||
Asset, AssetOptions, BundledAsset, CssAssetOptions, FolderAssetOptions, ImageAssetOptions,
|
||||
ImageFormat, ImageSize, JsAssetOptions,
|
||||
};
|
||||
@@ -0,0 +1,144 @@
|
||||
pub use const_serialize;
|
||||
use const_serialize::{serialize_const, ConstStr, ConstVec};
|
||||
use manganis_core::{AssetOptions, BundledAsset};
|
||||
|
||||
use crate::hash::ConstHasher;
|
||||
|
||||
/// Format the input path with a hash to create an unique output path for the macro in the form `{input_path}-{hash}.{extension}`
|
||||
pub const fn generate_unique_path(
|
||||
input_path: &str,
|
||||
content_hash: u64,
|
||||
asset_config: &AssetOptions,
|
||||
) -> ConstStr {
|
||||
// Format the unique path with the format `{input_path}-{hash}.{extension}`
|
||||
// Start with the input path
|
||||
let mut input_path = ConstStr::new(input_path);
|
||||
// Then strip the prefix from the input path. The path comes from the build platform, but
|
||||
// in wasm, we don't know what the path separator is from the build platform. We need to
|
||||
// split by both unix and windows paths and take the smallest one
|
||||
let mut extension = None;
|
||||
match (input_path.rsplit_once('/'), input_path.rsplit_once('\\')) {
|
||||
(Some((_, unix_new_input_path)), Some((_, windows_new_input_path))) => {
|
||||
input_path = if unix_new_input_path.len() < windows_new_input_path.len() {
|
||||
unix_new_input_path
|
||||
} else {
|
||||
windows_new_input_path
|
||||
};
|
||||
}
|
||||
(Some((_, unix_new_input_path)), _) => {
|
||||
input_path = unix_new_input_path;
|
||||
}
|
||||
(_, Some((_, windows_new_input_path))) => {
|
||||
input_path = windows_new_input_path;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
if let Some((new_input_path, new_extension)) = input_path.rsplit_once('.') {
|
||||
extension = Some(new_extension);
|
||||
input_path = new_input_path;
|
||||
}
|
||||
// Then add a dash
|
||||
let mut macro_output_path = input_path.push_str("-");
|
||||
|
||||
// Hash the contents along with the asset config to create a unique hash for the asset
|
||||
// When this hash changes, the client needs to re-fetch the asset
|
||||
let mut hasher = ConstHasher::new();
|
||||
hasher = hasher.write(&content_hash.to_le_bytes());
|
||||
hasher = hasher.hash_by_bytes(asset_config);
|
||||
let hash = hasher.finish();
|
||||
|
||||
// Then add the hash in hex form
|
||||
let hash_bytes = hash.to_le_bytes();
|
||||
let mut i = 0;
|
||||
while i < hash_bytes.len() {
|
||||
let byte = hash_bytes[i];
|
||||
let first = byte >> 4;
|
||||
let second = byte & 0x0f;
|
||||
const fn byte_to_char(byte: u8) -> char {
|
||||
match char::from_digit(byte as u32, 16) {
|
||||
Some(c) => c,
|
||||
None => panic!("byte must be a valid digit"),
|
||||
}
|
||||
}
|
||||
macro_output_path = macro_output_path.push(byte_to_char(first));
|
||||
macro_output_path = macro_output_path.push(byte_to_char(second));
|
||||
i += 1;
|
||||
}
|
||||
|
||||
// Finally add the extension
|
||||
match asset_config.extension() {
|
||||
Some(extension) => {
|
||||
macro_output_path = macro_output_path.push('.');
|
||||
macro_output_path = macro_output_path.push_str(extension)
|
||||
}
|
||||
None => {
|
||||
if let Some(extension) = extension {
|
||||
macro_output_path = macro_output_path.push('.');
|
||||
macro_output_path = macro_output_path.push_str(extension.as_str())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
macro_output_path
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_unique_path() {
|
||||
use manganis_core::{ImageAssetOptions, ImageFormat};
|
||||
use std::path::PathBuf;
|
||||
let mut input_path = PathBuf::from("some");
|
||||
input_path.push("prefix");
|
||||
input_path.push("test.png");
|
||||
let content_hash = 123456789;
|
||||
let asset_config = AssetOptions::Image(ImageAssetOptions::new().with_format(ImageFormat::Avif));
|
||||
let output_path =
|
||||
generate_unique_path(&input_path.to_string_lossy(), content_hash, &asset_config);
|
||||
assert_eq!(output_path.as_str(), "test-603a88fe296462a3.avif");
|
||||
|
||||
// Changing the path without changing the contents shouldn't change the hash
|
||||
let mut input_path = PathBuf::from("some");
|
||||
input_path.push("prefix");
|
||||
input_path.push("prefix");
|
||||
input_path.push("test.png");
|
||||
let content_hash = 123456789;
|
||||
let asset_config = AssetOptions::Image(ImageAssetOptions::new().with_format(ImageFormat::Avif));
|
||||
let output_path =
|
||||
generate_unique_path(&input_path.to_string_lossy(), content_hash, &asset_config);
|
||||
assert_eq!(output_path.as_str(), "test-603a88fe296462a3.avif");
|
||||
|
||||
let mut input_path = PathBuf::from("test");
|
||||
input_path.push("ing");
|
||||
input_path.push("test");
|
||||
let content_hash = 123456789;
|
||||
let asset_config = AssetOptions::Unknown;
|
||||
let output_path =
|
||||
generate_unique_path(&input_path.to_string_lossy(), content_hash, &asset_config);
|
||||
assert_eq!(output_path.as_str(), "test-c8c4cfad21cac262");
|
||||
|
||||
// Just changing the content hash should change the total hash
|
||||
let mut input_path = PathBuf::from("test");
|
||||
input_path.push("ing");
|
||||
input_path.push("test");
|
||||
let content_hash = 123456780;
|
||||
let asset_config = AssetOptions::Unknown;
|
||||
let output_path =
|
||||
generate_unique_path(&input_path.to_string_lossy(), content_hash, &asset_config);
|
||||
assert_eq!(output_path.as_str(), "test-7bced03789ff865c");
|
||||
}
|
||||
|
||||
/// Serialize an asset to a const buffer
|
||||
pub const fn serialize_asset(asset: &BundledAsset) -> ConstVec<u8> {
|
||||
let write = ConstVec::new();
|
||||
serialize_const(asset, write)
|
||||
}
|
||||
|
||||
/// Copy a slice into a constant sized buffer at compile time
|
||||
pub const fn copy_bytes<const N: usize>(bytes: &[u8]) -> [u8; N] {
|
||||
let mut out = [0; N];
|
||||
let mut i = 0;
|
||||
while i < N {
|
||||
out[i] = bytes[i];
|
||||
i += 1;
|
||||
}
|
||||
out
|
||||
}
|
||||
Reference in new issue
Block a user