Switch const serialize to use CBOR (#4932)

* implement const cbor encoding + decoding for basic types

* implement map encoding and decoding

* fix map test

* all tests passing

* dynamically sized arrays

* smaller str encoding

* swap array and list and add more documentation

* move serialization and deserialization for each struct together

* support both legacy and new assets

* bump const serialize version

* fix asset conversion code

* pull out manganis symbol struct from tuble

* test reading old asset versions

* more list/array cleanup and bump rust version

* fix clippy

* switch cli opt port

* install playwright deps

* split cli-optimization package to avoid two dioxus' in the same crate

* fix clippy

* force the toast.png asset to be included in the linker stage #[used] has platform depedant behavior

* bundle both formats in the link section and only use the asset the cli changed

* remove sudo which doesn't exist on windows

* add binstall to the playwright workflow

* separate bin target for the old cli

* lockfile

* cargo update to get versions shaken out

* notes on binstalls, add back `.read()` for compat

* use 0.8.0-alpha.0 for const-serialize instead of 0.8

this ensures we can still publish breaking changes on 0.8 without taking over the full version

* fix clippy in example

* small debug nits

* update rust-overlay in nix

* fix flake?

* migrate to new structure

* just comment out binstalls

* just remove old test harness

---------

Co-authored-by: Jonathan Kelley <jkelleyrtp@gmail.com>

packages/manganis/manganis-core/Cargo.toml

@@ -14,13 +14,13 @@ keywords = ["dom", "ui", "gui", "assets"]
 # prevent the main code from compiling!
 [dependencies]
 serde = { workspace = true, features = ["derive"] }
-const-serialize = { workspace = true, features = ["serde"] }
+const-serialize-08 = { workspace = true, features = ["serde", "const-serialize-07"] }
+const-serialize-07 = { workspace = true }
 dioxus-core-types = { workspace = true, optional = true }
 dioxus-cli-config = { workspace = true, optional = true }
 
 [dev-dependencies]
 manganis = { workspace = true }
-dioxus = { workspace = true }
 
 [features]
 dioxus = ["dep:dioxus-core-types", "dep:dioxus-cli-config"]

packages/manganis/manganis-core/src/asset.rs

@@ -1,5 +1,6 @@
 use crate::AssetOptions;
-use const_serialize::{deserialize_const, ConstStr, ConstVec, SerializeConst};
+use const_serialize_07 as const_serialize;
+use const_serialize_08::{deserialize_const, ConstStr, SerializeConst};
 use std::{fmt::Debug, hash::Hash, path::PathBuf};
 
 /// An asset that should be copied by the bundler with some options. This type will be
@@ -7,7 +8,17 @@ use std::{fmt::Debug, hash::Hash, path::PathBuf};
 /// CLIs that support manganis, should pull out the assets from the link section, optimize,
 /// and write them to the filesystem at [`BundledAsset::bundled_path`] for the application
 /// to use.
-#[derive(Debug, Eq, Clone, Copy, SerializeConst, serde::Serialize, serde::Deserialize)]
+#[derive(
+    Debug,
+    Eq,
+    Clone,
+    Copy,
+    SerializeConst,
+    const_serialize::SerializeConst,
+    serde::Serialize,
+    serde::Deserialize,
+)]
+#[const_serialize(crate = const_serialize_08)]
 pub struct BundledAsset {
     /// The absolute path of the asset
     absolute_source_path: ConstStr,
@@ -51,6 +62,8 @@ impl Hash for BundledAsset {
 }
 
 impl BundledAsset {
+    pub const PLACEHOLDER_HASH: &str = "This should be replaced by dx as part of the build process. If you see this error, make sure you are using a matching version of dx and dioxus and you are not stripping symbols from your binary.";
+
     #[doc(hidden)]
     /// This should only be called from the macro
     /// Create a new asset
@@ -86,7 +99,7 @@ impl BundledAsset {
 /// It should not be read directly with [`std::fs::read`] because the path needs to be resolved
 /// relative to the bundle
 ///
-/// ```rust
+/// ```rust, ignore
 /// # use manganis::{asset, Asset};
 /// # use dioxus::prelude::*;
 /// const ASSET: Asset = asset!("/assets/image.png");
@@ -99,13 +112,15 @@ impl BundledAsset {
 pub struct Asset {
     /// A function that returns a pointer to the bundled asset. This will be resolved after the linker has run and
     /// put into the lazy asset. We use a function instead of using the pointer directly to force the compiler to
-    /// read the static __REFERENCE_TO_LINK_SECTION at runtime which will be offset by the hot reloading engine instead
+    /// read the static __LINK_SECTION at runtime which will be offset by the hot reloading engine instead
     /// of at compile time which can't be offset
     ///
     /// WARNING: Don't read this directly. Reads can get optimized away at compile time before
     /// the data for this is filled in by the CLI after the binary is built. Instead, use
     /// [`std::ptr::read_volatile`] to read the data.
     bundled: fn() -> &'static [u8],
+    /// The legacy version of [`Self::bundled`]. This is only used for backwards compatibility with older versions of the CLI
+    legacy: fn() -> &'static [u8],
 }
 
 impl Debug for Asset {
@@ -121,27 +136,48 @@ impl Asset {
     #[doc(hidden)]
     /// This should only be called from the macro
     /// Create a new asset from the bundled form of the asset and the link section
-    pub const fn new(bundled: extern "Rust" fn() -> &'static [u8]) -> Self {
-        Self { bundled }
+    pub const fn new(
+        bundled: extern "Rust" fn() -> &'static [u8],
+        legacy: extern "Rust" fn() -> &'static [u8],
+    ) -> Self {
+        Self { bundled, legacy }
     }
 
     /// Get the bundled asset
     pub fn bundled(&self) -> BundledAsset {
+        // Read the slice using volatile reads to prevent the compiler from optimizing
+        // away the read at compile time
+        fn read_slice_volatile(bundled: &'static [u8]) -> const_serialize_07::ConstVec<u8> {
+            let ptr = bundled as *const [u8] as *const u8;
+            let len = bundled.len();
+            if ptr.is_null() {
+                panic!("Tried to use an asset that was not bundled. Make sure you are compiling dx as the linker");
+            }
+            let mut bytes = const_serialize_07::ConstVec::new();
+            for byte in 0..len {
+                // SAFETY: We checked that the pointer was not null above. The pointer is valid for reads and
+                // since we are reading a u8 there are no alignment requirements
+                let byte = unsafe { std::ptr::read_volatile(ptr.add(byte)) };
+                bytes = bytes.push(byte);
+            }
+            bytes
+        }
+
         let bundled = (self.bundled)();
-        let len = bundled.len();
-        let ptr = bundled as *const [u8] as *const u8;
-        if ptr.is_null() {
-            panic!("Tried to use an asset that was not bundled. Make sure you are compiling dx as the linker");
+        let bytes = read_slice_volatile(bundled);
+        let read = bytes.as_ref();
+        let asset = deserialize_const!(BundledAsset, read).expect("Failed to deserialize asset. Make sure you built with the matching version of the Dioxus CLI").1;
+
+        // If the asset wasn't bundled with the newer format, try the legacy format
+        if asset.bundled_path() == BundledAsset::PLACEHOLDER_HASH {
+            let bundled = (self.legacy)();
+            let bytes = read_slice_volatile(bundled);
+            let read = bytes.read();
+            let asset = const_serialize_07::deserialize_const!(BundledAsset, read).expect("Failed to deserialize asset. Make sure you built with the matching version of the Dioxus CLI").1;
+            asset
+        } else {
+            asset
         }
-        let mut bytes = ConstVec::new();
-        for byte in 0..len {
-            // SAFETY: We checked that the pointer was not null above. The pointer is valid for reads and
-            // since we are reading a u8 there are no alignment requirements
-            let byte = unsafe { std::ptr::read_volatile(ptr.add(byte)) };
-            bytes = bytes.push(byte);
-        }
-        let read = bytes.read();
-        deserialize_const!(BundledAsset, read).expect("Failed to deserialize asset. Make sure you built with the matching version of the Dioxus CLI").1
     }
 
     /// Return a canonicalized path to the asset

packages/manganis/manganis-core/src/css.rs

@@ -1,5 +1,6 @@
 use crate::{AssetOptions, AssetOptionsBuilder, AssetVariant};
-use const_serialize::SerializeConst;
+use const_serialize_07 as const_serialize;
+use const_serialize_08::SerializeConst;
 
 /// Options for a css asset
 #[derive(
@@ -11,9 +12,11 @@ use const_serialize::SerializeConst;
     Copy,
     Hash,
     SerializeConst,
+    const_serialize::SerializeConst,
     serde::Serialize,
     serde::Deserialize,
 )]
+#[const_serialize(crate = const_serialize_08)]
 pub struct CssAssetOptions {
     minify: bool,
     preload: bool,

packages/manganis/manganis-core/src/css_module.rs

@@ -1,5 +1,6 @@
 use crate::{AssetOptions, AssetOptionsBuilder, AssetVariant};
-use const_serialize::SerializeConst;
+use const_serialize_07 as const_serialize;
+use const_serialize_08::SerializeConst;
 use std::collections::HashSet;
 
 /// Options for a css module asset
@@ -12,9 +13,11 @@ use std::collections::HashSet;
     Copy,
     Hash,
     SerializeConst,
+    const_serialize::SerializeConst,
     serde::Serialize,
     serde::Deserialize,
 )]
+#[const_serialize(crate = const_serialize_08)]
 #[non_exhaustive]
 #[doc(hidden)]
 pub struct CssModuleAssetOptions {

packages/manganis/manganis-core/src/folder.rs

@@ -1,4 +1,5 @@
-use const_serialize::SerializeConst;
+use const_serialize_07 as const_serialize;
+use const_serialize_08::SerializeConst;
 
 use crate::{AssetOptions, AssetOptionsBuilder};
 
@@ -12,9 +13,11 @@ use crate::{AssetOptions, AssetOptionsBuilder};
     Copy,
     Hash,
     SerializeConst,
+    const_serialize::SerializeConst,
     serde::Serialize,
     serde::Deserialize,
 )]
+#[const_serialize(crate = const_serialize_08)]
 pub struct FolderAssetOptions {}
 
 impl Default for FolderAssetOptions {

packages/manganis/manganis-core/src/images.rs

@@ -1,4 +1,5 @@
-use const_serialize::SerializeConst;
+use const_serialize_07 as const_serialize;
+use const_serialize_08::SerializeConst;
 
 use crate::{AssetOptions, AssetOptionsBuilder, AssetVariant};
 
@@ -12,9 +13,11 @@ use crate::{AssetOptions, AssetOptionsBuilder, AssetVariant};
     Copy,
     Hash,
     SerializeConst,
+    const_serialize::SerializeConst,
     serde::Serialize,
     serde::Deserialize,
 )]
+#[const_serialize(crate = const_serialize_08)]
 #[repr(u8)]
 pub enum ImageFormat {
     /// A png image. Png images cannot contain transparency and tend to compress worse than other formats
@@ -39,9 +42,11 @@ pub enum ImageFormat {
     Copy,
     Hash,
     SerializeConst,
+    const_serialize::SerializeConst,
     serde::Serialize,
     serde::Deserialize,
 )]
+#[const_serialize(crate = const_serialize_08)]
 #[repr(C, u8)]
 pub enum ImageSize {
     /// A manual size in pixels
@@ -65,9 +70,11 @@ pub enum ImageSize {
     Copy,
     Hash,
     SerializeConst,
+    const_serialize::SerializeConst,
     serde::Serialize,
     serde::Deserialize,
 )]
+#[const_serialize(crate = const_serialize_08)]
 pub struct ImageAssetOptions {
     ty: ImageFormat,
     low_quality_preview: bool,

packages/manganis/manganis-core/src/js.rs

@@ -1,4 +1,5 @@
-use const_serialize::SerializeConst;
+use const_serialize_07 as const_serialize;
+use const_serialize_08::SerializeConst;
 
 use crate::{AssetOptions, AssetOptionsBuilder, AssetVariant};
 
@@ -12,9 +13,11 @@ use crate::{AssetOptions, AssetOptionsBuilder, AssetVariant};
     Copy,
     Hash,
     SerializeConst,
+    const_serialize::SerializeConst,
     serde::Serialize,
     serde::Deserialize,
 )]
+#[const_serialize(crate = const_serialize_08)]
 pub struct JsAssetOptions {
     minify: bool,
     preload: bool,

packages/manganis/manganis-core/src/options.rs

@@ -1,4 +1,5 @@
-use const_serialize::SerializeConst;
+use const_serialize_07 as const_serialize;
+use const_serialize_08::SerializeConst;
 
 use crate::{
     CssAssetOptions, CssModuleAssetOptions, FolderAssetOptions, ImageAssetOptions, JsAssetOptions,
@@ -14,9 +15,11 @@ use crate::{
     Copy,
     Hash,
     SerializeConst,
+    const_serialize::SerializeConst,
     serde::Serialize,
     serde::Deserialize,
 )]
+#[const_serialize(crate = const_serialize_08)]
 #[non_exhaustive]
 pub struct AssetOptions {
     /// If a hash should be added to the asset path
@@ -107,7 +110,7 @@ impl AssetOptionsBuilder<()> {
 
 impl<T> AssetOptionsBuilder<T> {
     /// Create a new asset options builder with the given variant
-    pub(crate) const fn variant(variant: T) -> Self {
+    pub const fn variant(variant: T) -> Self {
         Self {
             add_hash: true,
             variant,
@@ -153,9 +156,11 @@ impl<T> AssetOptionsBuilder<T> {
     Copy,
     Hash,
     SerializeConst,
+    const_serialize::SerializeConst,
     serde::Serialize,
     serde::Deserialize,
 )]
+#[const_serialize(crate = const_serialize_08)]
 #[repr(C, u8)]
 #[non_exhaustive]
 pub enum AssetVariant {

packages/manganis/manganis-macro/src/asset.rs

@@ -119,9 +119,10 @@ impl AssetParser {
 
                 #link_section
 
-                static __REFERENCE_TO_LINK_SECTION: &'static [u8] = &__LINK_SECTION;
-
-                manganis::Asset::new(|| unsafe { std::ptr::read_volatile(&__REFERENCE_TO_LINK_SECTION) })
+                manganis::Asset::new(
+                    || unsafe { std::ptr::read_volatile(&__LINK_SECTION) },
+                    || unsafe { std::ptr::read_volatile(&__LEGACY_LINK_SECTION) }
+                )
             }
         }
     }

packages/manganis/manganis-macro/src/linker.rs

@@ -8,18 +8,37 @@ use quote::ToTokens;
 /// After linking, the "manganis" sections of the different object files will be merged.
 pub fn generate_link_section(asset: impl ToTokens, asset_hash: &str) -> TokenStream2 {
     let position = proc_macro2::Span::call_site();
-    let export_name = syn::LitStr::new(&format!("__MANGANIS__{}", asset_hash), position);
+    let export_name = syn::LitStr::new(&format!("__ASSETS__{}", asset_hash), position);
+    let legacy_export_name = syn::LitStr::new(&format!("__MANGANIS__{}", asset_hash), position);
 
     quote::quote! {
-        // First serialize the asset into a constant sized buffer
-        const __BUFFER: manganis::macro_helpers::const_serialize::ConstVec<u8> = manganis::macro_helpers::serialize_asset(&#asset);
-        // Then pull out the byte slice
-        const __BYTES: &[u8] = __BUFFER.as_ref();
-        // And the length of the byte slice
-        const __LEN: usize = __BYTES.len();
+        // We bundle both the legacy and new link sections for compatibility with older CLIs
+        static __LEGACY_LINK_SECTION: &'static [u8] = {
+            // First serialize the asset into a constant sized buffer
+            const __BUFFER: manganis::macro_helpers::const_serialize_07::ConstVec<u8> = manganis::macro_helpers::serialize_asset_07(&#asset);
+            // Then pull out the byte slice
+            const __BYTES: &[u8] = __BUFFER.as_ref();
+            // And the length of the byte slice
+            const __LEN: usize = __BYTES.len();
 
-        // Now that we have the size of the asset, copy the bytes into a static array
-        #[unsafe(export_name = #export_name)]
-        static __LINK_SECTION: [u8; __LEN]  = manganis::macro_helpers::copy_bytes(__BYTES);
+            // Now that we have the size of the asset, copy the bytes into a static array
+            #[unsafe(export_name = #legacy_export_name)]
+            static __LINK_SECTION: [u8; __LEN]  = manganis::macro_helpers::copy_bytes(__BYTES);
+            &__LINK_SECTION
+        };
+
+        static __LINK_SECTION: &'static [u8] = {
+            // First serialize the asset into a constant sized buffer
+            const __BUFFER: manganis::macro_helpers::const_serialize::ConstVec<u8> = manganis::macro_helpers::serialize_asset(&#asset);
+            // Then pull out the byte slice
+            const __BYTES: &[u8] = __BUFFER.as_ref();
+            // And the length of the byte slice
+            const __LEN: usize = __BYTES.len();
+
+            // Now that we have the size of the asset, copy the bytes into a static array
+            #[unsafe(export_name = #export_name)]
+            static __LINK_SECTION: [u8; __LEN]  = manganis::macro_helpers::copy_bytes(__BYTES);
+            &__LINK_SECTION
+        };
     }
 }

packages/manganis/manganis/Cargo.toml

@@ -14,6 +14,7 @@ keywords = ["assets"]
 
 [dependencies]
 const-serialize = { workspace = true }
+const-serialize-07 = { workspace = true }
 manganis-core = { workspace = true }
 manganis-macro = { workspace = true }
 

packages/manganis/manganis/src/macro_helpers.rs

@@ -1,12 +1,15 @@
 pub use const_serialize;
 use const_serialize::{serialize_const, ConstVec, SerializeConst};
+pub use const_serialize_07;
+use const_serialize_07::{
+    serialize_const as serialize_const_07, ConstVec as ConstVec07,
+    SerializeConst as SerializeConst07,
+};
 use manganis_core::{AssetOptions, BundledAsset};
 
-const PLACEHOLDER_HASH: &str = "This should be replaced by dx as part of the build process. If you see this error, make sure you are using a matching version of dx and dioxus and you are not stripping symbols from your binary.";
-
 /// Create a bundled asset from the input path, the content hash, and the asset options
 pub const fn create_bundled_asset(input_path: &str, asset_config: AssetOptions) -> BundledAsset {
-    BundledAsset::new(input_path, PLACEHOLDER_HASH, asset_config)
+    BundledAsset::new(input_path, BundledAsset::PLACEHOLDER_HASH, asset_config)
 }
 
 /// Create a bundled asset from the input path, the content hash, and the asset options with a relative asset deprecation warning
@@ -27,7 +30,18 @@ pub const fn serialize_asset(asset: &BundledAsset) -> ConstVec<u8> {
     let data = ConstVec::new();
     let mut data = serialize_const(asset, data);
     // Reserve the maximum size of the asset
-    while data.len() < BundledAsset::MEMORY_LAYOUT.size() {
+    while data.len() < <BundledAsset as SerializeConst>::MEMORY_LAYOUT.size() {
+        data = data.push(0);
+    }
+    data
+}
+
+/// Serialize an asset to a const buffer in the legacy 0.7 format
+pub const fn serialize_asset_07(asset: &BundledAsset) -> ConstVec07<u8> {
+    let data = ConstVec07::new();
+    let mut data = serialize_const_07(asset, data);
+    // Reserve the maximum size of the asset
+    while data.len() < <BundledAsset as SerializeConst07>::MEMORY_LAYOUT.size() {
         data = data.push(0);
     }
     data
@@ -35,8 +49,7 @@ pub const fn serialize_asset(asset: &BundledAsset) -> ConstVec<u8> {
 
 /// Deserialize a const buffer into a BundledAsset
 pub const fn deserialize_asset(bytes: &[u8]) -> BundledAsset {
-    let bytes = ConstVec::new().extend(bytes);
-    match const_serialize::deserialize_const!(BundledAsset, bytes.read()) {
+    match const_serialize::deserialize_const!(BundledAsset, bytes) {
         Some((_, asset)) => asset,
         None => panic!("Failed to deserialize asset. This may be caused by a mismatch between your dioxus and dioxus-cli versions"),
     }