Switch const serialize to use CBOR (#4932)

* implement const cbor encoding + decoding for basic types

* implement map encoding and decoding

* fix map test

* all tests passing

* dynamically sized arrays

* smaller str encoding

* swap array and list and add more documentation

* move serialization and deserialization for each struct together

* support both legacy and new assets

* bump const serialize version

* fix asset conversion code

* pull out manganis symbol struct from tuble

* test reading old asset versions

* more list/array cleanup and bump rust version

* fix clippy

* switch cli opt port

* install playwright deps

* split cli-optimization package to avoid two dioxus' in the same crate

* fix clippy

* force the toast.png asset to be included in the linker stage #[used] has platform depedant behavior

* bundle both formats in the link section and only use the asset the cli changed

* remove sudo which doesn't exist on windows

* add binstall to the playwright workflow

* separate bin target for the old cli

* lockfile

* cargo update to get versions shaken out

* notes on binstalls, add back `.read()` for compat

* use 0.8.0-alpha.0 for const-serialize instead of 0.8

this ensures we can still publish breaking changes on 0.8 without taking over the full version

* fix clippy in example

* small debug nits

* update rust-overlay in nix

* fix flake?

* migrate to new structure

* just comment out binstalls

* just remove old test harness

---------

Co-authored-by: Jonathan Kelley <jkelleyrtp@gmail.com>

packages/manganis/manganis-core/src/asset.rs

@@ -1,5 +1,6 @@
 use crate::AssetOptions;
-use const_serialize::{deserialize_const, ConstStr, ConstVec, SerializeConst};
+use const_serialize_07 as const_serialize;
+use const_serialize_08::{deserialize_const, ConstStr, SerializeConst};
 use std::{fmt::Debug, hash::Hash, path::PathBuf};
 
 /// An asset that should be copied by the bundler with some options. This type will be
@@ -7,7 +8,17 @@ use std::{fmt::Debug, hash::Hash, path::PathBuf};
 /// CLIs that support manganis, should pull out the assets from the link section, optimize,
 /// and write them to the filesystem at [`BundledAsset::bundled_path`] for the application
 /// to use.
-#[derive(Debug, Eq, Clone, Copy, SerializeConst, serde::Serialize, serde::Deserialize)]
+#[derive(
+    Debug,
+    Eq,
+    Clone,
+    Copy,
+    SerializeConst,
+    const_serialize::SerializeConst,
+    serde::Serialize,
+    serde::Deserialize,
+)]
+#[const_serialize(crate = const_serialize_08)]
 pub struct BundledAsset {
     /// The absolute path of the asset
     absolute_source_path: ConstStr,
@@ -51,6 +62,8 @@ impl Hash for BundledAsset {
 }
 
 impl BundledAsset {
+    pub const PLACEHOLDER_HASH: &str = "This should be replaced by dx as part of the build process. If you see this error, make sure you are using a matching version of dx and dioxus and you are not stripping symbols from your binary.";
+
     #[doc(hidden)]
     /// This should only be called from the macro
     /// Create a new asset
@@ -86,7 +99,7 @@ impl BundledAsset {
 /// It should not be read directly with [`std::fs::read`] because the path needs to be resolved
 /// relative to the bundle
 ///
-/// ```rust
+/// ```rust, ignore
 /// # use manganis::{asset, Asset};
 /// # use dioxus::prelude::*;
 /// const ASSET: Asset = asset!("/assets/image.png");
@@ -99,13 +112,15 @@ impl BundledAsset {
 pub struct Asset {
     /// A function that returns a pointer to the bundled asset. This will be resolved after the linker has run and
     /// put into the lazy asset. We use a function instead of using the pointer directly to force the compiler to
-    /// read the static __REFERENCE_TO_LINK_SECTION at runtime which will be offset by the hot reloading engine instead
+    /// read the static __LINK_SECTION at runtime which will be offset by the hot reloading engine instead
     /// of at compile time which can't be offset
     ///
     /// WARNING: Don't read this directly. Reads can get optimized away at compile time before
     /// the data for this is filled in by the CLI after the binary is built. Instead, use
     /// [`std::ptr::read_volatile`] to read the data.
     bundled: fn() -> &'static [u8],
+    /// The legacy version of [`Self::bundled`]. This is only used for backwards compatibility with older versions of the CLI
+    legacy: fn() -> &'static [u8],
 }
 
 impl Debug for Asset {
@@ -121,27 +136,48 @@ impl Asset {
     #[doc(hidden)]
     /// This should only be called from the macro
     /// Create a new asset from the bundled form of the asset and the link section
-    pub const fn new(bundled: extern "Rust" fn() -> &'static [u8]) -> Self {
-        Self { bundled }
+    pub const fn new(
+        bundled: extern "Rust" fn() -> &'static [u8],
+        legacy: extern "Rust" fn() -> &'static [u8],
+    ) -> Self {
+        Self { bundled, legacy }
     }
 
     /// Get the bundled asset
     pub fn bundled(&self) -> BundledAsset {
+        // Read the slice using volatile reads to prevent the compiler from optimizing
+        // away the read at compile time
+        fn read_slice_volatile(bundled: &'static [u8]) -> const_serialize_07::ConstVec<u8> {
+            let ptr = bundled as *const [u8] as *const u8;
+            let len = bundled.len();
+            if ptr.is_null() {
+                panic!("Tried to use an asset that was not bundled. Make sure you are compiling dx as the linker");
+            }
+            let mut bytes = const_serialize_07::ConstVec::new();
+            for byte in 0..len {
+                // SAFETY: We checked that the pointer was not null above. The pointer is valid for reads and
+                // since we are reading a u8 there are no alignment requirements
+                let byte = unsafe { std::ptr::read_volatile(ptr.add(byte)) };
+                bytes = bytes.push(byte);
+            }
+            bytes
+        }
+
         let bundled = (self.bundled)();
-        let len = bundled.len();
-        let ptr = bundled as *const [u8] as *const u8;
-        if ptr.is_null() {
-            panic!("Tried to use an asset that was not bundled. Make sure you are compiling dx as the linker");
+        let bytes = read_slice_volatile(bundled);
+        let read = bytes.as_ref();
+        let asset = deserialize_const!(BundledAsset, read).expect("Failed to deserialize asset. Make sure you built with the matching version of the Dioxus CLI").1;
+
+        // If the asset wasn't bundled with the newer format, try the legacy format
+        if asset.bundled_path() == BundledAsset::PLACEHOLDER_HASH {
+            let bundled = (self.legacy)();
+            let bytes = read_slice_volatile(bundled);
+            let read = bytes.read();
+            let asset = const_serialize_07::deserialize_const!(BundledAsset, read).expect("Failed to deserialize asset. Make sure you built with the matching version of the Dioxus CLI").1;
+            asset
+        } else {
+            asset
         }
-        let mut bytes = ConstVec::new();
-        for byte in 0..len {
-            // SAFETY: We checked that the pointer was not null above. The pointer is valid for reads and
-            // since we are reading a u8 there are no alignment requirements
-            let byte = unsafe { std::ptr::read_volatile(ptr.add(byte)) };
-            bytes = bytes.push(byte);
-        }
-        let read = bytes.read();
-        deserialize_const!(BundledAsset, read).expect("Failed to deserialize asset. Make sure you built with the matching version of the Dioxus CLI").1
     }
 
     /// Return a canonicalized path to the asset

packages/manganis/manganis-core/src/css.rs

@@ -1,5 +1,6 @@
 use crate::{AssetOptions, AssetOptionsBuilder, AssetVariant};
-use const_serialize::SerializeConst;
+use const_serialize_07 as const_serialize;
+use const_serialize_08::SerializeConst;
 
 /// Options for a css asset
 #[derive(
@@ -11,9 +12,11 @@ use const_serialize::SerializeConst;
     Copy,
     Hash,
     SerializeConst,
+    const_serialize::SerializeConst,
     serde::Serialize,
     serde::Deserialize,
 )]
+#[const_serialize(crate = const_serialize_08)]
 pub struct CssAssetOptions {
     minify: bool,
     preload: bool,

packages/manganis/manganis-core/src/css_module.rs

@@ -1,5 +1,6 @@
 use crate::{AssetOptions, AssetOptionsBuilder, AssetVariant};
-use const_serialize::SerializeConst;
+use const_serialize_07 as const_serialize;
+use const_serialize_08::SerializeConst;
 use std::collections::HashSet;
 
 /// Options for a css module asset
@@ -12,9 +13,11 @@ use std::collections::HashSet;
     Copy,
     Hash,
     SerializeConst,
+    const_serialize::SerializeConst,
     serde::Serialize,
     serde::Deserialize,
 )]
+#[const_serialize(crate = const_serialize_08)]
 #[non_exhaustive]
 #[doc(hidden)]
 pub struct CssModuleAssetOptions {

packages/manganis/manganis-core/src/folder.rs

@@ -1,4 +1,5 @@
-use const_serialize::SerializeConst;
+use const_serialize_07 as const_serialize;
+use const_serialize_08::SerializeConst;
 
 use crate::{AssetOptions, AssetOptionsBuilder};
 
@@ -12,9 +13,11 @@ use crate::{AssetOptions, AssetOptionsBuilder};
     Copy,
     Hash,
     SerializeConst,
+    const_serialize::SerializeConst,
     serde::Serialize,
     serde::Deserialize,
 )]
+#[const_serialize(crate = const_serialize_08)]
 pub struct FolderAssetOptions {}
 
 impl Default for FolderAssetOptions {

packages/manganis/manganis-core/src/images.rs

@@ -1,4 +1,5 @@
-use const_serialize::SerializeConst;
+use const_serialize_07 as const_serialize;
+use const_serialize_08::SerializeConst;
 
 use crate::{AssetOptions, AssetOptionsBuilder, AssetVariant};
 
@@ -12,9 +13,11 @@ use crate::{AssetOptions, AssetOptionsBuilder, AssetVariant};
     Copy,
     Hash,
     SerializeConst,
+    const_serialize::SerializeConst,
     serde::Serialize,
     serde::Deserialize,
 )]
+#[const_serialize(crate = const_serialize_08)]
 #[repr(u8)]
 pub enum ImageFormat {
     /// A png image. Png images cannot contain transparency and tend to compress worse than other formats
@@ -39,9 +42,11 @@ pub enum ImageFormat {
     Copy,
     Hash,
     SerializeConst,
+    const_serialize::SerializeConst,
     serde::Serialize,
     serde::Deserialize,
 )]
+#[const_serialize(crate = const_serialize_08)]
 #[repr(C, u8)]
 pub enum ImageSize {
     /// A manual size in pixels
@@ -65,9 +70,11 @@ pub enum ImageSize {
     Copy,
     Hash,
     SerializeConst,
+    const_serialize::SerializeConst,
     serde::Serialize,
     serde::Deserialize,
 )]
+#[const_serialize(crate = const_serialize_08)]
 pub struct ImageAssetOptions {
     ty: ImageFormat,
     low_quality_preview: bool,

packages/manganis/manganis-core/src/js.rs

@@ -1,4 +1,5 @@
-use const_serialize::SerializeConst;
+use const_serialize_07 as const_serialize;
+use const_serialize_08::SerializeConst;
 
 use crate::{AssetOptions, AssetOptionsBuilder, AssetVariant};
 
@@ -12,9 +13,11 @@ use crate::{AssetOptions, AssetOptionsBuilder, AssetVariant};
     Copy,
     Hash,
     SerializeConst,
+    const_serialize::SerializeConst,
     serde::Serialize,
     serde::Deserialize,
 )]
+#[const_serialize(crate = const_serialize_08)]
 pub struct JsAssetOptions {
     minify: bool,
     preload: bool,

packages/manganis/manganis-core/src/options.rs

@@ -1,4 +1,5 @@
-use const_serialize::SerializeConst;
+use const_serialize_07 as const_serialize;
+use const_serialize_08::SerializeConst;
 
 use crate::{
     CssAssetOptions, CssModuleAssetOptions, FolderAssetOptions, ImageAssetOptions, JsAssetOptions,
@@ -14,9 +15,11 @@ use crate::{
     Copy,
     Hash,
     SerializeConst,
+    const_serialize::SerializeConst,
     serde::Serialize,
     serde::Deserialize,
 )]
+#[const_serialize(crate = const_serialize_08)]
 #[non_exhaustive]
 pub struct AssetOptions {
     /// If a hash should be added to the asset path
@@ -107,7 +110,7 @@ impl AssetOptionsBuilder<()> {
 
 impl<T> AssetOptionsBuilder<T> {
     /// Create a new asset options builder with the given variant
-    pub(crate) const fn variant(variant: T) -> Self {
+    pub const fn variant(variant: T) -> Self {
         Self {
             add_hash: true,
             variant,
@@ -153,9 +156,11 @@ impl<T> AssetOptionsBuilder<T> {
     Copy,
     Hash,
     SerializeConst,
+    const_serialize::SerializeConst,
     serde::Serialize,
     serde::Deserialize,
 )]
+#[const_serialize(crate = const_serialize_08)]
 #[repr(C, u8)]
 #[non_exhaustive]
 pub enum AssetVariant {