Restore manganis optimizations (#3195)
* Create a crate for constant serialization of config structs for manganis * use SerializeConst for the image asset builder * switch to a declarative macro for assets * add serializable options for each asset type * implement the new manganis macro * optimize assets in the CLI * Fix assets with dioxus formattable * reduce fuzzing test limits * use the new syntax in examples * fix avif support * Fix manganis doc tests * cache asset optimization * Split out asset and bundled asset * make hash pointer width independent * Fix manganis macro docs * add constvec::is_empty method to fix clippy lint * remove nasm feature * simplify test_rsplit_once test so typos passes * revert example change from clippy fix * always accept unix paths * fix windows path seperator * fix folder asset hash * Optimize assets in a blocking task * Fix asset options docs * Document Asset and BundledAsset * simplify the linker macro a bit * add more docs to AssetParser expansion * add image format helpers * Split by both unix and windows path separators and take the smallest one
This commit is contained in:
Evan Almloff authored and GitHub committed
1 parent
31d72db82d
commit
f9617c8d33
64 files changed
+6233
-704
No files matched your search
@@ -1,14 +1,107 @@
|
||||
use manganis_core::{AssetParseError, ResourceAsset};
|
||||
use proc_macro2::TokenStream as TokenStream2;
|
||||
use quote::{quote, ToTokens, TokenStreamExt};
|
||||
use std::{
|
||||
hash::Hasher,
|
||||
io::Read,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
use syn::{
|
||||
parse::{Parse, ParseStream},
|
||||
LitStr,
|
||||
LitStr, Token,
|
||||
};
|
||||
|
||||
fn resolve_path(raw: &str) -> Result<PathBuf, AssetParseError> {
|
||||
// Get the location of the root of the crate which is where all assets are relative to
|
||||
//
|
||||
// IE
|
||||
// /users/dioxus/dev/app/
|
||||
// is the root of
|
||||
// /users/dioxus/dev/app/assets/blah.css
|
||||
let manifest_dir = std::env::var("CARGO_MANIFEST_DIR")
|
||||
.map(PathBuf::from)
|
||||
.unwrap();
|
||||
|
||||
// 1. the input file should be a pathbuf
|
||||
let input = PathBuf::from(raw);
|
||||
|
||||
// 2. absolute path to the asset
|
||||
manifest_dir
|
||||
.join(raw.trim_start_matches('/'))
|
||||
.canonicalize()
|
||||
.map_err(|err| AssetParseError::AssetDoesntExist {
|
||||
err,
|
||||
path: input.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
fn hash_file_contents(file_path: &Path) -> Result<u64, AssetParseError> {
|
||||
// Create a hasher
|
||||
let mut hash = std::collections::hash_map::DefaultHasher::new();
|
||||
|
||||
// If this is a folder, hash the folder contents
|
||||
if file_path.is_dir() {
|
||||
let files =
|
||||
std::fs::read_dir(file_path).map_err(|err| AssetParseError::AssetDoesntExist {
|
||||
err,
|
||||
path: file_path.to_path_buf(),
|
||||
})?;
|
||||
for file in files.flatten() {
|
||||
let path = file.path();
|
||||
hash_file_contents(&path)?;
|
||||
}
|
||||
return Ok(hash.finish());
|
||||
}
|
||||
|
||||
// Otherwise, open the file to get its contents
|
||||
let mut file =
|
||||
std::fs::File::open(file_path).map_err(|err| AssetParseError::AssetDoesntExist {
|
||||
err,
|
||||
path: file_path.to_path_buf(),
|
||||
})?;
|
||||
|
||||
// We add a hash to the end of the file so it is invalidated when the bundled version of the file changes
|
||||
// The hash includes the file contents, the options, and the version of manganis. From the macro, we just
|
||||
// know the file contents, so we only include that hash
|
||||
let mut buffer = [0; 8192];
|
||||
loop {
|
||||
let read = file
|
||||
.read(&mut buffer)
|
||||
.map_err(AssetParseError::FailedToReadAsset)?;
|
||||
if read == 0 {
|
||||
break;
|
||||
}
|
||||
hash.write(&buffer[..read]);
|
||||
}
|
||||
|
||||
Ok(hash.finish())
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum AssetParseError {
|
||||
AssetDoesntExist {
|
||||
err: std::io::Error,
|
||||
path: std::path::PathBuf,
|
||||
},
|
||||
FailedToReadAsset(std::io::Error),
|
||||
}
|
||||
|
||||
impl std::fmt::Display for AssetParseError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
AssetParseError::AssetDoesntExist { err, path } => {
|
||||
write!(f, "Asset at {} doesn't exist: {}", path.display(), err)
|
||||
}
|
||||
AssetParseError::FailedToReadAsset(err) => write!(f, "Failed to read asset: {}", err),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct AssetParser {
|
||||
/// The span of the source string
|
||||
path_span: proc_macro2::Span,
|
||||
|
||||
/// The asset itself
|
||||
asset: Result<ResourceAsset, AssetParseError>,
|
||||
asset: Result<PathBuf, AssetParseError>,
|
||||
|
||||
/// The source of the trailing options
|
||||
options: TokenStream2,
|
||||
@@ -26,8 +119,8 @@ impl Parse for AssetParser {
|
||||
// ```
|
||||
// asset!(
|
||||
// "/assets/myfile.png",
|
||||
// asset::image()
|
||||
// .format(ImageType::Jpg)
|
||||
// ImageAssetOptions::new()
|
||||
// .format(ImageFormat::Jpg)
|
||||
// .size(512, 512)
|
||||
// )
|
||||
// ```
|
||||
@@ -36,29 +129,27 @@ impl Parse for AssetParser {
|
||||
fn parse(input: ParseStream) -> syn::Result<Self> {
|
||||
// And then parse the options
|
||||
let src = input.parse::<LitStr>()?;
|
||||
let asset = ResourceAsset::parse_any(&src.value());
|
||||
let path_span = src.span();
|
||||
let asset = resolve_path(&src.value());
|
||||
let _comma = input.parse::<Token![,]>();
|
||||
let options = input.parse()?;
|
||||
|
||||
Ok(Self { asset, options })
|
||||
Ok(Self {
|
||||
path_span,
|
||||
asset,
|
||||
options,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl ToTokens for AssetParser {
|
||||
// Need to generate:
|
||||
//
|
||||
// - 1. absolute file path on the user's system: `/users/dioxus/dev/project/assets/blah.css`
|
||||
// - 2. original input in case that's useful: `../blah.css`
|
||||
// - 3. path relative to the CARGO_MANIFEST_DIR - and then we'll add a `/`: `/assets/blah.css
|
||||
// - 4. file from which this macro was called: `/users/dioxus/dev/project/src/lib.rs`
|
||||
// - 5: The link section containing all this data
|
||||
// - 6: the input tokens such that the builder gets validated by the const code
|
||||
// - 7: the bundled name `/blahcss123.css`
|
||||
//
|
||||
// Not that we'll use everything, but at least we have this metadata for more post-processing.
|
||||
//
|
||||
// For now, `2` and `3` will be the same since we don't support relative paths... a bit of
|
||||
// a limitation from rust itself. We technically could support them but not without some hoops
|
||||
// to jump through
|
||||
// The manganis macro outputs info to two different places:
|
||||
// 1) The crate the macro was invoked in
|
||||
// - It needs the hashed contents of the file, the file path, and the file options
|
||||
// - Most of this is just forwarding the input, the only thing that the macro needs to do is hash the file contents
|
||||
// 2) A bundler that supports manganis (currently just dioxus-cli)
|
||||
// - The macro needs to output the absolute path to the asset for the bundler to find later
|
||||
// - It also needs to serialize the bundled asset along with the asset options for the bundler to use later
|
||||
fn to_tokens(&self, tokens: &mut proc_macro2::TokenStream) {
|
||||
let asset = match self.asset.as_ref() {
|
||||
Ok(asset) => asset,
|
||||
@@ -68,47 +159,60 @@ impl ToTokens for AssetParser {
|
||||
return;
|
||||
}
|
||||
};
|
||||
let asset_str = asset.display().to_string();
|
||||
let mut asset_str = proc_macro2::Literal::string(&asset_str);
|
||||
asset_str.set_span(self.path_span);
|
||||
|
||||
// 1. the link section itself
|
||||
let link_section = crate::generate_link_section(&asset);
|
||||
let hash = match hash_file_contents(asset) {
|
||||
Ok(hash) => hash,
|
||||
Err(err) => {
|
||||
let err = err.to_string();
|
||||
tokens.append_all(quote! { compile_error!(#err) });
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// 2. original
|
||||
let input = asset.input.display().to_string();
|
||||
|
||||
// 3. resolved on the user's system
|
||||
let local = asset.absolute.display().to_string();
|
||||
|
||||
// 4. bundled
|
||||
let bundled = asset.bundled.to_string();
|
||||
|
||||
// 5. source tokens
|
||||
let option_source = &self.options;
|
||||
// Generate the link section for the asset
|
||||
// The link section includes the source path and the output path of the asset
|
||||
let link_section = crate::generate_link_section(quote!(__ASSET));
|
||||
|
||||
// generate the asset::new method to deprecate the `./assets/blah.css` syntax
|
||||
let method = if !asset.input.starts_with("/") {
|
||||
let constructor = if asset.is_relative() {
|
||||
quote::quote! { new_relative }
|
||||
} else {
|
||||
quote::quote! { new }
|
||||
};
|
||||
|
||||
let options = if self.options.is_empty() {
|
||||
quote! { manganis::AssetOptions::Unknown }
|
||||
} else {
|
||||
self.options.clone()
|
||||
};
|
||||
|
||||
tokens.extend(quote! {
|
||||
Asset::#method(
|
||||
{
|
||||
#link_section
|
||||
manganis::Asset {
|
||||
// "/assets/blah.css"
|
||||
input: #input,
|
||||
{
|
||||
// We keep a hash of the contents of the asset for cache busting
|
||||
const __ASSET_HASH: u64 = #hash;
|
||||
// The source is used by the CLI to copy the asset
|
||||
const __ASSET_SOURCE_PATH: &'static str = #asset_str;
|
||||
// The options give the CLI info about how to process the asset
|
||||
// Note: into_asset_options is not a trait, so we cannot accept the options directly
|
||||
// in the constructor. Stable rust doesn't have support for constant functions in traits
|
||||
const __ASSET_OPTIONS: manganis::AssetOptions = #options.into_asset_options();
|
||||
// We calculate the bundled path from the hash and any transformations done by the options
|
||||
// This is the final path that the asset will be written to
|
||||
const __ASSET_BUNDLED_PATH: manganis::macro_helpers::const_serialize::ConstStr = manganis::macro_helpers::generate_unique_path(__ASSET_SOURCE_PATH, __ASSET_HASH, &__ASSET_OPTIONS);
|
||||
// Get the reference to the string that was generated. We cannot return &'static str from
|
||||
// generate_unique_path because it would return a reference to data generated in the function
|
||||
const __ASSET_BUNDLED_PATH_STR: &'static str = __ASSET_BUNDLED_PATH.as_str();
|
||||
// Create the asset that the crate will use. This is used both in the return value and
|
||||
// added to the linker for the bundler to copy later
|
||||
const __ASSET: manganis::BundledAsset = manganis::BundledAsset::#constructor(__ASSET_SOURCE_PATH, __ASSET_BUNDLED_PATH_STR, __ASSET_OPTIONS);
|
||||
|
||||
// "/users/dioxus/dev/app/assets/blah.css"
|
||||
local: #local,
|
||||
#link_section
|
||||
|
||||
// "/blahcss123.css"
|
||||
bundled: #bundled,
|
||||
|
||||
metadata: __volatile_reader as fn() -> u8,
|
||||
}
|
||||
}
|
||||
) #option_source
|
||||
manganis::Asset::new(__ASSET, __keep_link_section)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,42 @@ pub(crate) mod linker;
|
||||
use linker::generate_link_section;
|
||||
|
||||
/// The asset macro collects assets that will be included in the final binary
|
||||
///
|
||||
/// # Files
|
||||
///
|
||||
/// The file builder collects an arbitrary file. Relative paths are resolved relative to the package root
|
||||
/// ```rust
|
||||
/// # use manganis::{asset, Asset};
|
||||
/// const _: Asset = asset!("/assets/asset.txt");
|
||||
/// ```
|
||||
/// Or you can use URLs to read the asset at build time from a remote location
|
||||
/// ```rust
|
||||
/// # use manganis::{asset, Asset};
|
||||
/// const _: Asset = asset!("/assets/image.png");
|
||||
/// ```
|
||||
///
|
||||
/// # Images
|
||||
///
|
||||
/// You can collect images which will be automatically optimized with the image builder:
|
||||
/// ```rust
|
||||
/// # use manganis::{asset, Asset};
|
||||
/// const _: Asset = asset!("/assets/image.png");
|
||||
/// ```
|
||||
/// Resize the image at compile time to make the assets file size smaller:
|
||||
/// ```rust
|
||||
/// # use manganis::{asset, Asset, ImageAssetOptions, ImageSize};
|
||||
/// const _: Asset = asset!("/assets/image.png", ImageAssetOptions::new().with_size(ImageSize::Manual { width: 52, height: 52 }));
|
||||
/// ```
|
||||
/// Or convert the image at compile time to a web friendly format:
|
||||
/// ```rust
|
||||
/// # use manganis::{asset, Asset, ImageAssetOptions, ImageSize, ImageFormat};
|
||||
/// const _: Asset = asset!("/assets/image.png", ImageAssetOptions::new().with_format(ImageFormat::Avif));
|
||||
/// ```
|
||||
/// You can mark images as preloaded to make them load faster in your app
|
||||
/// ```rust
|
||||
/// # use manganis::{asset, Asset, ImageAssetOptions};
|
||||
/// const _: Asset = asset!("/assets/image.png", ImageAssetOptions::new().with_preload(true));
|
||||
/// ```
|
||||
#[proc_macro]
|
||||
pub fn asset(input: TokenStream) -> TokenStream {
|
||||
let asset = parse_macro_input!(input as asset::AssetParser);
|
||||
|
||||
@@ -1,25 +1,33 @@
|
||||
use proc_macro2::TokenStream as TokenStream2;
|
||||
use serde::Serialize;
|
||||
use quote::ToTokens;
|
||||
|
||||
/// this new approach will store the assets descriptions *inside the executable*.
|
||||
/// The trick is to use the `link_section` attribute.
|
||||
/// We force rust to store a json representation of the asset description
|
||||
/// We store description of the assets an application uses in the executable.
|
||||
/// We use the `link_section` attribute embed an extra section in the executable.
|
||||
/// We force rust to store a serialized representation of the asset description
|
||||
/// inside a particular region of the binary, with the label "manganis".
|
||||
/// After linking, the "manganis" sections of the different executables will be merged.
|
||||
pub fn generate_link_section(asset: &impl Serialize) -> TokenStream2 {
|
||||
/// After linking, the "manganis" sections of the different object files will be merged.
|
||||
pub fn generate_link_section(asset: impl ToTokens) -> TokenStream2 {
|
||||
let position = proc_macro2::Span::call_site();
|
||||
let asset_description = serde_json::to_string(asset).unwrap();
|
||||
let len = asset_description.as_bytes().len();
|
||||
let asset_bytes = syn::LitByteStr::new(asset_description.as_bytes(), position);
|
||||
let section_name = syn::LitStr::new(manganis_core::LinkSection::CURRENT.link_section, position);
|
||||
let section_name = syn::LitStr::new(
|
||||
manganis_core::linker::LinkSection::CURRENT.link_section,
|
||||
position,
|
||||
);
|
||||
|
||||
quote::quote! {
|
||||
// First serialize the asset into a constant sized buffer
|
||||
const __BUFFER: manganis::macro_helpers::const_serialize::ConstVec<u8> = manganis::macro_helpers::serialize_asset(&#asset);
|
||||
// Then pull out the byte slice
|
||||
const __BYTES: &[u8] = __BUFFER.as_ref();
|
||||
// And the length of the byte slice
|
||||
const __LEN: usize = __BYTES.len();
|
||||
|
||||
// Now that we have the size of the asset, copy the bytes into a static array
|
||||
#[link_section = #section_name]
|
||||
#[used]
|
||||
static ASSET: [u8; #len] = * #asset_bytes;
|
||||
static __LINK_SECTION: [u8; __LEN] = manganis::macro_helpers::copy_bytes(__BYTES);
|
||||
|
||||
fn __volatile_reader() -> u8 {
|
||||
unsafe { std::ptr::read_volatile(ASSET.as_ptr()) }
|
||||
fn __keep_link_section() -> u8 {
|
||||
unsafe { std::ptr::read_volatile(__LINK_SECTION.as_ptr()) }
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user