Switch const serialize to use CBOR (#4932)

* implement const cbor encoding + decoding for basic types

* implement map encoding and decoding

* fix map test

* all tests passing

* dynamically sized arrays

* smaller str encoding

* swap array and list and add more documentation

* move serialization and deserialization for each struct together

* support both legacy and new assets

* bump const serialize version

* fix asset conversion code

* pull out manganis symbol struct from tuble

* test reading old asset versions

* more list/array cleanup and bump rust version

* fix clippy

* switch cli opt port

* install playwright deps

* split cli-optimization package to avoid two dioxus' in the same crate

* fix clippy

* force the toast.png asset to be included in the linker stage #[used] has platform depedant behavior

* bundle both formats in the link section and only use the asset the cli changed

* remove sudo which doesn't exist on windows

* add binstall to the playwright workflow

* separate bin target for the old cli

* lockfile

* cargo update to get versions shaken out

* notes on binstalls, add back `.read()` for compat

* use 0.8.0-alpha.0 for const-serialize instead of 0.8

this ensures we can still publish breaking changes on 0.8 without taking over the full version

* fix clippy in example

* small debug nits

* update rust-overlay in nix

* fix flake?

* migrate to new structure

* just comment out binstalls

* just remove old test harness

---------

Co-authored-by: Jonathan Kelley <jkelleyrtp@gmail.com>

packages/const-serialize/tests/enum.rs

@@ -81,7 +81,7 @@ fn test_serialize_enum() {
     let mut buf = ConstVec::new();
     buf = serialize_const(&data, buf);
     println!("{:?}", buf.as_ref());
-    let buf = buf.read();
+    let buf = buf.as_ref();
     assert_eq!(deserialize_const!(Enum, buf).unwrap().1, data);
 
     let data = Enum::B {
@@ -91,7 +91,7 @@ fn test_serialize_enum() {
     let mut buf = ConstVec::new();
     buf = serialize_const(&data, buf);
     println!("{:?}", buf.as_ref());
-    let buf = buf.read();
+    let buf = buf.as_ref();
     assert_eq!(deserialize_const!(Enum, buf).unwrap().1, data);
 }
 
@@ -110,7 +110,7 @@ fn test_serialize_list_of_lopsided_enums() {
     let mut buf = ConstVec::new();
     buf = serialize_const(&data, buf);
     println!("{:?}", buf.as_ref());
-    let buf = buf.read();
+    let buf = buf.as_ref();
     assert_eq!(deserialize_const!([Enum; 2], buf).unwrap().1, data);
 
     let data = [
@@ -126,7 +126,7 @@ fn test_serialize_list_of_lopsided_enums() {
     let mut buf = ConstVec::new();
     buf = serialize_const(&data, buf);
     println!("{:?}", buf.as_ref());
-    let buf = buf.read();
+    let buf = buf.as_ref();
     assert_eq!(deserialize_const!([Enum; 2], buf).unwrap().1, data);
 
     let data = [
@@ -139,7 +139,7 @@ fn test_serialize_list_of_lopsided_enums() {
     let mut buf = ConstVec::new();
     buf = serialize_const(&data, buf);
     println!("{:?}", buf.as_ref());
-    let buf = buf.read();
+    let buf = buf.as_ref();
     assert_eq!(deserialize_const!([Enum; 2], buf).unwrap().1, data);
 
     let data = [
@@ -152,7 +152,7 @@ fn test_serialize_list_of_lopsided_enums() {
     let mut buf = ConstVec::new();
     buf = serialize_const(&data, buf);
     println!("{:?}", buf.as_ref());
-    let buf = buf.read();
+    let buf = buf.as_ref();
     assert_eq!(deserialize_const!([Enum; 2], buf).unwrap().1, data);
 }
 
@@ -171,14 +171,14 @@ fn test_serialize_u8_enum() {
     let mut buf = ConstVec::new();
     buf = serialize_const(&data, buf);
     println!("{:?}", buf.as_ref());
-    let buf = buf.read();
+    let buf = buf.as_ref();
     assert_eq!(deserialize_const!(Enum, buf).unwrap().1, data);
 
     let data = Enum::B;
     let mut buf = ConstVec::new();
     buf = serialize_const(&data, buf);
     println!("{:?}", buf.as_ref());
-    let buf = buf.read();
+    let buf = buf.as_ref();
     assert_eq!(deserialize_const!(Enum, buf).unwrap().1, data);
 }
 
@@ -198,7 +198,7 @@ fn test_serialize_corrupted_enum() {
     buf = serialize_const(&data, buf);
     buf = buf.set(0, 2);
     println!("{:?}", buf.as_ref());
-    let buf = buf.read();
+    let buf = buf.as_ref();
     assert_eq!(deserialize_const!(Enum, buf), None);
 }
 
@@ -226,7 +226,7 @@ fn test_serialize_nested_enum() {
     let mut buf = ConstVec::new();
     buf = serialize_const(&data, buf);
     println!("{:?}", buf.as_ref());
-    let buf = buf.read();
+    let buf = buf.as_ref();
     assert_eq!(deserialize_const!(Enum, buf).unwrap().1, data);
 
     let data = Enum::B {
@@ -236,7 +236,7 @@ fn test_serialize_nested_enum() {
     let mut buf = ConstVec::new();
     buf = serialize_const(&data, buf);
     println!("{:?}", buf.as_ref());
-    let buf = buf.read();
+    let buf = buf.as_ref();
     assert_eq!(deserialize_const!(Enum, buf).unwrap().1, data);
 
     let data = Enum::B {
@@ -249,7 +249,7 @@ fn test_serialize_nested_enum() {
     let mut buf = ConstVec::new();
     buf = serialize_const(&data, buf);
     println!("{:?}", buf.as_ref());
-    let buf = buf.read();
+    let buf = buf.as_ref();
     assert_eq!(deserialize_const!(Enum, buf).unwrap().1, data);
 
     let data = Enum::B {
@@ -262,6 +262,81 @@ fn test_serialize_nested_enum() {
     let mut buf = ConstVec::new();
     buf = serialize_const(&data, buf);
     println!("{:?}", buf.as_ref());
-    let buf = buf.read();
+    let buf = buf.as_ref();
     assert_eq!(deserialize_const!(Enum, buf).unwrap().1, data);
 }
+
+#[test]
+fn test_adding_enum_field_non_breaking() {
+    #[derive(Debug, PartialEq, SerializeConst)]
+    #[repr(C, u8)]
+    enum Initial {
+        A { a: u32, b: u8 },
+    }
+
+    #[derive(Debug, PartialEq, SerializeConst)]
+    #[repr(C, u8)]
+    enum New {
+        A { b: u8, a: u32, c: u32 },
+    }
+
+    let data = New::A {
+        a: 0x11111111,
+        b: 0x22,
+        c: 0x33333333,
+    };
+    let mut buf = ConstVec::new();
+    buf = serialize_const(&data, buf);
+    let buf = buf.as_ref();
+    // The new struct should be able to deserialize into the initial struct
+    let (_, data2) = deserialize_const!(Initial, buf).unwrap();
+    assert_eq!(
+        Initial::A {
+            a: 0x11111111,
+            b: 0x22,
+        },
+        data2
+    );
+}
+
+#[test]
+fn test_adding_enum_variant_non_breaking() {
+    #[derive(Debug, PartialEq, SerializeConst)]
+    #[repr(C, u8)]
+    enum Initial {
+        A { a: u32, b: u8 },
+    }
+
+    #[derive(Debug, PartialEq, SerializeConst)]
+    #[repr(C, u8)]
+    enum New {
+        #[allow(unused)]
+        B {
+            d: u32,
+            e: u8,
+        },
+        A {
+            c: u32,
+            b: u8,
+            a: u32,
+        },
+    }
+
+    let data = New::A {
+        a: 0x11111111,
+        b: 0x22,
+        c: 0x33333333,
+    };
+    let mut buf = ConstVec::new();
+    buf = serialize_const(&data, buf);
+    let buf = buf.as_ref();
+    // The new struct should be able to deserialize into the initial struct
+    let (_, data2) = deserialize_const!(Initial, buf).unwrap();
+    assert_eq!(
+        Initial::A {
+            a: 0x11111111,
+            b: 0x22,
+        },
+        data2
+    );
+}

packages/const-serialize/tests/lists.rs

@@ -5,7 +5,7 @@ fn test_serialize_const_layout_list() {
     let mut buf = ConstVec::new();
     buf = serialize_const(&[1u8, 2, 3] as &[u8; 3], buf);
     println!("{:?}", buf.as_ref());
-    let buf = buf.read();
+    let buf = buf.as_ref();
     assert_eq!(deserialize_const!([u8; 3], buf).unwrap().1, [1, 2, 3])
 }
 
@@ -17,7 +17,7 @@ fn test_serialize_const_layout_nested_lists() {
         buf,
     );
     println!("{:?}", buf.as_ref());
-    let buf = buf.read();
+    let buf = buf.as_ref();
 
     assert_eq!(
         deserialize_const!([[u8; 3]; 3], buf).unwrap().1,
@@ -29,6 +29,6 @@ fn test_serialize_const_layout_nested_lists() {
 fn test_serialize_list_too_little_data() {
     let mut buf = ConstVec::new();
     buf = buf.push(1);
-    let buf = buf.read();
+    let buf = buf.as_ref();
     assert_eq!(deserialize_const!([u64; 10], buf), None);
 }

packages/const-serialize/tests/primitive.rs

@@ -4,58 +4,33 @@ use const_serialize::{deserialize_const, serialize_const, ConstVec};
 fn test_serialize_const_layout_primitive() {
     let mut buf = ConstVec::new();
     buf = serialize_const(&1234u32, buf);
-    if cfg!(feature = "test-big-endian") {
-        assert_eq!(buf.as_ref(), 1234u32.to_be_bytes());
-    } else {
-        assert_eq!(buf.as_ref(), 1234u32.to_le_bytes());
-    }
-    let buf = buf.read();
+    let buf = buf.as_ref();
     assert_eq!(deserialize_const!(u32, buf).unwrap().1, 1234u32);
 
     let mut buf = ConstVec::new();
     buf = serialize_const(&1234u64, buf);
-    if cfg!(feature = "test-big-endian") {
-        assert_eq!(buf.as_ref(), 1234u64.to_be_bytes());
-    } else {
-        assert_eq!(buf.as_ref(), 1234u64.to_le_bytes());
-    }
-    let buf = buf.read();
+    let buf = buf.as_ref();
     assert_eq!(deserialize_const!(u64, buf).unwrap().1, 1234u64);
 
     let mut buf = ConstVec::new();
     buf = serialize_const(&1234i32, buf);
-    if cfg!(feature = "test-big-endian") {
-        assert_eq!(buf.as_ref(), 1234i32.to_be_bytes());
-    } else {
-        assert_eq!(buf.as_ref(), 1234i32.to_le_bytes());
-    }
-    let buf = buf.read();
+    let buf = buf.as_ref();
     assert_eq!(deserialize_const!(i32, buf).unwrap().1, 1234i32);
 
     let mut buf = ConstVec::new();
     buf = serialize_const(&1234i64, buf);
-    if cfg!(feature = "test-big-endian") {
-        assert_eq!(buf.as_ref(), 1234i64.to_be_bytes());
-    } else {
-        assert_eq!(buf.as_ref(), 1234i64.to_le_bytes());
-    }
-    let buf = buf.read();
+    let buf = buf.as_ref();
     assert_eq!(deserialize_const!(i64, buf).unwrap().1, 1234i64);
 
     let mut buf = ConstVec::new();
     buf = serialize_const(&true, buf);
     assert_eq!(buf.as_ref(), [1u8]);
-    let buf = buf.read();
+    let buf = buf.as_ref();
     assert!(deserialize_const!(bool, buf).unwrap().1);
 
     let mut buf = ConstVec::new();
     buf = serialize_const(&0.631f32, buf);
-    if cfg!(feature = "test-big-endian") {
-        assert_eq!(buf.as_ref(), 0.631f32.to_be_bytes());
-    } else {
-        assert_eq!(buf.as_ref(), 0.631f32.to_le_bytes());
-    }
-    let buf = buf.read();
+    let buf = buf.as_ref();
     assert_eq!(deserialize_const!(f32, buf).unwrap().1, 0.631);
 }
 
@@ -66,6 +41,6 @@ fn test_serialize_primitive_too_little_data() {
     buf = buf.push(1);
     buf = buf.push(1);
     buf = buf.push(1);
-    let buf = buf.read();
-    assert_eq!(deserialize_const!(u64, buf), None);
+    let buf = buf.as_ref();
+    assert_eq!(deserialize_const!([u64; 10], buf), None);
 }

packages/const-serialize/tests/str.rs

@@ -6,11 +6,10 @@ fn test_serialize_const_layout_str() {
     let str = ConstStr::new("hello");
     buf = serialize_const(&str, buf);
     println!("{:?}", buf.as_ref());
-    let buf = buf.read();
-    assert_eq!(
-        deserialize_const!(ConstStr, buf).unwrap().1.as_str(),
-        "hello"
-    );
+    let buf = buf.as_ref();
+    assert!(buf.len() < 10);
+    let str = deserialize_const!(ConstStr, buf).unwrap().1;
+    assert_eq!(str.as_str(), "hello");
 }
 
 #[test]
@@ -19,7 +18,8 @@ fn test_serialize_const_layout_nested_str() {
     let str = ConstStr::new("hello");
     buf = serialize_const(&[str, str, str] as &[ConstStr; 3], buf);
     println!("{:?}", buf.as_ref());
-    let buf = buf.read();
+    assert!(buf.len() < 30);
+    let buf = buf.as_ref();
 
     assert_eq!(
         deserialize_const!([ConstStr; 3], buf).unwrap().1,
@@ -35,6 +35,6 @@ fn test_serialize_const_layout_nested_str() {
 fn test_serialize_str_too_little_data() {
     let mut buf = ConstVec::new();
     buf = buf.push(1);
-    let buf = buf.read();
+    let buf = buf.as_ref();
     assert_eq!(deserialize_const!(ConstStr, buf), None);
 }

packages/const-serialize/tests/structs.rs

@@ -96,7 +96,7 @@ fn test_serialize_const_layout_struct_list() {
     const _ASSERT: () = {
         let mut buf = ConstVec::new();
         buf = serialize_const(&DATA, buf);
-        let buf = buf.read();
+        let buf = buf.as_ref();
         let [first, second, third] = match deserialize_const!([OtherStruct; 3], buf) {
             Some((_, data)) => data,
             None => panic!("data mismatch"),
@@ -109,7 +109,7 @@ fn test_serialize_const_layout_struct_list() {
         let mut buf = ConstVec::new();
         const DATA_AGAIN: [[OtherStruct; 3]; 3] = [DATA, DATA, DATA];
         buf = serialize_const(&DATA_AGAIN, buf);
-        let buf = buf.read();
+        let buf = buf.as_ref();
         let [first, second, third] = match deserialize_const!([[OtherStruct; 3]; 3], buf) {
             Some((_, data)) => data,
             None => panic!("data mismatch"),
@@ -128,7 +128,7 @@ fn test_serialize_const_layout_struct_list() {
     let mut buf = ConstVec::new();
     buf = serialize_const(&DATA, buf);
     println!("{:?}", buf.as_ref());
-    let buf = buf.read();
+    let buf = buf.as_ref();
     let (_, data2) = deserialize_const!([OtherStruct; 3], buf).unwrap();
     assert_eq!(DATA, data2);
 }
@@ -158,7 +158,41 @@ fn test_serialize_const_layout_struct() {
     let mut buf = ConstVec::new();
     buf = serialize_const(&data, buf);
     println!("{:?}", buf.as_ref());
-    let buf = buf.read();
+    let buf = buf.as_ref();
     let (_, data2) = deserialize_const!(OtherStruct, buf).unwrap();
     assert_eq!(data, data2);
 }
+
+#[test]
+fn test_adding_struct_field_non_breaking() {
+    #[derive(Debug, PartialEq, SerializeConst)]
+    struct Initial {
+        a: u32,
+        b: u8,
+    }
+
+    #[derive(Debug, PartialEq, SerializeConst)]
+    struct New {
+        c: u32,
+        b: u8,
+        a: u32,
+    }
+
+    let data = New {
+        a: 0x11111111,
+        b: 0x22,
+        c: 0x33333333,
+    };
+    let mut buf = ConstVec::new();
+    buf = serialize_const(&data, buf);
+    let buf = buf.as_ref();
+    // The new struct should be able to deserialize into the initial struct
+    let (_, data2) = deserialize_const!(Initial, buf).unwrap();
+    assert_eq!(
+        Initial {
+            a: data.a,
+            b: data.b,
+        },
+        data2
+    );
+}

packages/const-serialize/tests/tuples.rs

@@ -4,7 +4,7 @@ use const_serialize::{deserialize_const, serialize_const, ConstVec};
 fn test_serialize_const_layout_tuple() {
     let mut buf = ConstVec::new();
     buf = serialize_const(&(1234u32, 5678u16), buf);
-    let buf = buf.read();
+    let buf = buf.as_ref();
     assert_eq!(
         deserialize_const!((u32, u16), buf).unwrap().1,
         (1234u32, 5678u16)
@@ -12,7 +12,7 @@ fn test_serialize_const_layout_tuple() {
 
     let mut buf = ConstVec::new();
     buf = serialize_const(&(1234f64, 5678u16, 90u8), buf);
-    let buf = buf.read();
+    let buf = buf.as_ref();
     assert_eq!(
         deserialize_const!((f64, u16, u8), buf).unwrap().1,
         (1234f64, 5678u16, 90u8)
@@ -20,7 +20,7 @@ fn test_serialize_const_layout_tuple() {
 
     let mut buf = ConstVec::new();
     buf = serialize_const(&(1234u32, 5678u16, 90u8, 1000000f64), buf);
-    let buf = buf.read();
+    let buf = buf.as_ref();
     assert_eq!(
         deserialize_const!((u32, u16, u8, f64), buf).unwrap().1,
         (1234u32, 5678u16, 90u8, 1000000f64)