Restore manganis optimizations (#3195)

* Create a crate for constant serialization of config structs for manganis
* use SerializeConst for the image asset builder
* switch to a declarative macro for assets
* add serializable options for each asset type
* implement the new manganis macro
* optimize assets in the CLI
* Fix assets with dioxus formattable
* reduce fuzzing test limits
* use the new syntax in examples
* fix avif support
* Fix manganis doc tests
* cache asset optimization
* Split out asset and bundled asset
* make hash pointer width independent
* Fix manganis macro docs
* add constvec::is_empty method to fix clippy lint
* remove nasm feature
* simplify test_rsplit_once test so typos passes
* revert example change from clippy fix
* always accept unix paths
* fix windows path seperator
* fix folder asset hash
* Optimize assets in a blocking task
* Fix asset options docs
* Document Asset and BundledAsset
* simplify the linker macro a bit
* add more docs to AssetParser expansion
* add image format helpers
* Split by  both unix and windows path separators and take the smallest one
This commit is contained in:
Evan Almloff authored and GitHub committed 2024-11-26 18:12:24 +01:00
1 parent 31d72db82d
commit f9617c8d33
64 files changed
+6233 -704

No files matched your search

+206
View File
@@ -0,0 +1,206 @@
use const_serialize::{deserialize_const, serialize_const, ConstVec, SerializeConst};
use std::mem::MaybeUninit;
#[test]
fn test_transmute_bytes_to_enum() {
#[derive(Clone, Copy, Debug, PartialEq)]
#[repr(C, u8)]
enum Enum<T> {
A { one: u32, two: u16 },
B { one: u8, two: T } = 15,
}
#[repr(C)]
#[derive(Debug, PartialEq)]
struct A {
one: u32,
two: u16,
}
#[repr(C)]
#[derive(Debug, PartialEq)]
struct B<T> {
one: u8,
two: T,
}
const SIZE: usize = std::mem::size_of::<Enum<u16>>();
let mut out = [MaybeUninit::uninit(); SIZE];
let discriminate_size = std::mem::size_of::<u8>();
let tag_align = 0;
let union_alignment = std::mem::align_of::<A>().max(std::mem::align_of::<B<u16>>());
let data_align = (discriminate_size / union_alignment) + union_alignment;
let a_one_align = std::mem::offset_of!(A, one);
let a_two_align = std::mem::offset_of!(A, two);
let b_one_align = std::mem::offset_of!(B<u16>, one);
let b_two_align = std::mem::offset_of!(B<u16>, two);
let one = 1234u32;
let two = 5678u16;
let first = Enum::A { one, two };
for (i, byte) in one.to_le_bytes().iter().enumerate() {
out[data_align + i + a_one_align] = MaybeUninit::new(*byte);
}
for (i, byte) in two.to_le_bytes().iter().enumerate() {
out[data_align + i + a_two_align] = MaybeUninit::new(*byte);
}
out[tag_align] = MaybeUninit::new(0);
let out = unsafe { std::mem::transmute_copy::<[MaybeUninit<u8>; SIZE], Enum<u16>>(&out) };
assert_eq!(out, first);
let mut out = [MaybeUninit::uninit(); SIZE];
let one = 123u8;
let two = 58u16;
let second = Enum::B { one, two };
for (i, byte) in one.to_le_bytes().iter().enumerate() {
out[data_align + i + b_one_align] = MaybeUninit::new(*byte);
}
for (i, byte) in two.to_le_bytes().iter().enumerate() {
out[data_align + i + b_two_align] = MaybeUninit::new(*byte);
}
out[tag_align] = MaybeUninit::new(15);
let out = unsafe { std::mem::transmute_copy::<[MaybeUninit<u8>; SIZE], Enum<u16>>(&out) };
assert_eq!(out, second);
}
#[test]
fn test_serialize_enum() {
#[derive(Clone, Copy, Debug, PartialEq, SerializeConst)]
#[repr(C, u8)]
enum Enum {
A { one: u32, two: u16 },
B { one: u8, two: u16 } = 15,
}
println!("{:#?}", Enum::MEMORY_LAYOUT);
let data = Enum::A {
one: 0x11111111,
two: 0x22,
};
let mut buf = ConstVec::new();
buf = serialize_const(&data, buf);
println!("{:?}", buf.as_ref());
let buf = buf.read();
assert_eq!(deserialize_const!(Enum, buf).unwrap().1, data);
let data = Enum::B {
one: 0x11,
two: 0x2233,
};
let mut buf = ConstVec::new();
buf = serialize_const(&data, buf);
println!("{:?}", buf.as_ref());
let buf = buf.read();
assert_eq!(deserialize_const!(Enum, buf).unwrap().1, data);
}
#[test]
fn test_serialize_u8_enum() {
#[derive(Clone, Copy, Debug, PartialEq, SerializeConst)]
#[repr(u8)]
enum Enum {
A,
B,
}
println!("{:#?}", Enum::MEMORY_LAYOUT);
let data = Enum::A;
let mut buf = ConstVec::new();
buf = serialize_const(&data, buf);
println!("{:?}", buf.as_ref());
let buf = buf.read();
assert_eq!(deserialize_const!(Enum, buf).unwrap().1, data);
let data = Enum::B;
let mut buf = ConstVec::new();
buf = serialize_const(&data, buf);
println!("{:?}", buf.as_ref());
let buf = buf.read();
assert_eq!(deserialize_const!(Enum, buf).unwrap().1, data);
}
#[test]
fn test_serialize_corrupted_enum() {
#[derive(Clone, Copy, Debug, PartialEq, SerializeConst)]
#[repr(C, u8)]
enum Enum {
A { one: u32, two: u16 },
}
let data = Enum::A {
one: 0x11111111,
two: 0x22,
};
let mut buf = ConstVec::new();
buf = serialize_const(&data, buf);
buf = buf.set(0, 2);
println!("{:?}", buf.as_ref());
let buf = buf.read();
assert_eq!(deserialize_const!(Enum, buf), None);
}
#[test]
fn test_serialize_nested_enum() {
#[derive(Clone, Copy, Debug, PartialEq, SerializeConst)]
#[repr(C, u8)]
enum Enum {
A { one: u32, two: u16 },
B { one: u8, two: InnerEnum } = 15,
}
#[derive(Clone, Copy, Debug, PartialEq, SerializeConst)]
#[repr(C, u16)]
enum InnerEnum {
A(u8),
B { one: u64, two: f64 } = 1000,
C { one: u32, two: u16 },
}
let data = Enum::A {
one: 0x11111111,
two: 0x22,
};
let mut buf = ConstVec::new();
buf = serialize_const(&data, buf);
println!("{:?}", buf.as_ref());
let buf = buf.read();
assert_eq!(deserialize_const!(Enum, buf).unwrap().1, data);
let data = Enum::B {
one: 0x11,
two: InnerEnum::A(0x22),
};
let mut buf = ConstVec::new();
buf = serialize_const(&data, buf);
println!("{:?}", buf.as_ref());
let buf = buf.read();
assert_eq!(deserialize_const!(Enum, buf).unwrap().1, data);
let data = Enum::B {
one: 0x11,
two: InnerEnum::B {
one: 0x2233,
two: 0.123456789,
},
};
let mut buf = ConstVec::new();
buf = serialize_const(&data, buf);
println!("{:?}", buf.as_ref());
let buf = buf.read();
assert_eq!(deserialize_const!(Enum, buf).unwrap().1, data);
let data = Enum::B {
one: 0x11,
two: InnerEnum::C {
one: 0x2233,
two: 56789,
},
};
let mut buf = ConstVec::new();
buf = serialize_const(&data, buf);
println!("{:?}", buf.as_ref());
let buf = buf.read();
assert_eq!(deserialize_const!(Enum, buf).unwrap().1, data);
}
+59
View File
@@ -0,0 +1,59 @@
use const_serialize::{serialize_eq, SerializeConst};
#[derive(Clone, Copy, Debug, PartialEq, SerializeConst)]
struct Struct {
a: u32,
b: u8,
c: u32,
d: Enum,
}
#[derive(Clone, Copy, Debug, PartialEq, SerializeConst)]
#[repr(C, u8)]
enum Enum {
A { one: u32, two: u16 },
B { one: u8, two: u16 } = 15,
}
#[test]
fn const_eq() {
const {
let data = [
Struct {
a: 0x11111111,
b: 0x22,
c: 0x33333333,
d: Enum::A {
one: 0x44444444,
two: 0x5555,
},
},
Struct {
a: 123,
b: 9,
c: 38,
d: Enum::B {
one: 0x44,
two: 0x555,
},
},
Struct {
a: 9,
b: 123,
c: 39,
d: Enum::B {
one: 0x46,
two: 0x555,
},
},
];
let mut other = data;
other[2].a += 1;
if serialize_eq(&data, &other) {
panic!("data should be different");
}
if !serialize_eq(&data, &data) {
panic!("data should be the same");
}
}
}
+34
View File
@@ -0,0 +1,34 @@
use const_serialize::{deserialize_const, serialize_const, ConstVec};
#[test]
fn test_serialize_const_layout_list() {
let mut buf = ConstVec::new();
buf = serialize_const(&[1u8, 2, 3] as &[u8; 3], buf);
println!("{:?}", buf.as_ref());
let buf = buf.read();
assert_eq!(deserialize_const!([u8; 3], buf).unwrap().1, [1, 2, 3])
}
#[test]
fn test_serialize_const_layout_nested_lists() {
let mut buf = ConstVec::new();
buf = serialize_const(
&[[1u8, 2, 3], [4u8, 5, 6], [7u8, 8, 9]] as &[[u8; 3]; 3],
buf,
);
println!("{:?}", buf.as_ref());
let buf = buf.read();
assert_eq!(
deserialize_const!([[u8; 3]; 3], buf).unwrap().1,
[[1, 2, 3], [4, 5, 6], [7, 8, 9]]
);
}
#[test]
fn test_serialize_list_too_little_data() {
let mut buf = ConstVec::new();
buf = buf.push(1);
let buf = buf.read();
assert_eq!(deserialize_const!([u64; 10], buf), None);
}
@@ -0,0 +1,71 @@
use const_serialize::{deserialize_const, serialize_const, ConstVec};
#[test]
fn test_serialize_const_layout_primitive() {
let mut buf = ConstVec::new();
buf = serialize_const(&1234u32, buf);
if cfg!(feature = "test-big-endian") {
assert_eq!(buf.as_ref(), 1234u32.to_be_bytes());
} else {
assert_eq!(buf.as_ref(), 1234u32.to_le_bytes());
}
let buf = buf.read();
assert_eq!(deserialize_const!(u32, buf).unwrap().1, 1234u32);
let mut buf = ConstVec::new();
buf = serialize_const(&1234u64, buf);
if cfg!(feature = "test-big-endian") {
assert_eq!(buf.as_ref(), 1234u64.to_be_bytes());
} else {
assert_eq!(buf.as_ref(), 1234u64.to_le_bytes());
}
let buf = buf.read();
assert_eq!(deserialize_const!(u64, buf).unwrap().1, 1234u64);
let mut buf = ConstVec::new();
buf = serialize_const(&1234i32, buf);
if cfg!(feature = "test-big-endian") {
assert_eq!(buf.as_ref(), 1234i32.to_be_bytes());
} else {
assert_eq!(buf.as_ref(), 1234i32.to_le_bytes());
}
let buf = buf.read();
assert_eq!(deserialize_const!(i32, buf).unwrap().1, 1234i32);
let mut buf = ConstVec::new();
buf = serialize_const(&1234i64, buf);
if cfg!(feature = "test-big-endian") {
assert_eq!(buf.as_ref(), 1234i64.to_be_bytes());
} else {
assert_eq!(buf.as_ref(), 1234i64.to_le_bytes());
}
let buf = buf.read();
assert_eq!(deserialize_const!(i64, buf).unwrap().1, 1234i64);
let mut buf = ConstVec::new();
buf = serialize_const(&true, buf);
assert_eq!(buf.as_ref(), [1u8]);
let buf = buf.read();
assert!(deserialize_const!(bool, buf).unwrap().1);
let mut buf = ConstVec::new();
buf = serialize_const(&0.631f32, buf);
if cfg!(feature = "test-big-endian") {
assert_eq!(buf.as_ref(), 0.631f32.to_be_bytes());
} else {
assert_eq!(buf.as_ref(), 0.631f32.to_le_bytes());
}
let buf = buf.read();
assert_eq!(deserialize_const!(f32, buf).unwrap().1, 0.631);
}
#[test]
fn test_serialize_primitive_too_little_data() {
let mut buf = ConstVec::new();
buf = buf.push(1);
buf = buf.push(1);
buf = buf.push(1);
let buf = buf.read();
assert_eq!(deserialize_const!(u64, buf), None);
}
+40
View File
@@ -0,0 +1,40 @@
use const_serialize::{deserialize_const, serialize_const, ConstStr, ConstVec};
#[test]
fn test_serialize_const_layout_str() {
let mut buf = ConstVec::new();
let str = ConstStr::new("hello");
buf = serialize_const(&str, buf);
println!("{:?}", buf.as_ref());
let buf = buf.read();
assert_eq!(
deserialize_const!(ConstStr, buf).unwrap().1.as_str(),
"hello"
);
}
#[test]
fn test_serialize_const_layout_nested_str() {
let mut buf = ConstVec::new();
let str = ConstStr::new("hello");
buf = serialize_const(&[str, str, str] as &[ConstStr; 3], buf);
println!("{:?}", buf.as_ref());
let buf = buf.read();
assert_eq!(
deserialize_const!([ConstStr; 3], buf).unwrap().1,
[
ConstStr::new("hello"),
ConstStr::new("hello"),
ConstStr::new("hello")
]
);
}
#[test]
fn test_serialize_str_too_little_data() {
let mut buf = ConstVec::new();
buf = buf.push(1);
let buf = buf.read();
assert_eq!(deserialize_const!(ConstStr, buf), None);
}
+164
View File
@@ -0,0 +1,164 @@
use const_serialize::{deserialize_const, serialize_const, ConstVec, SerializeConst};
use std::mem::MaybeUninit;
#[test]
fn test_transmute_bytes_to_struct() {
struct MyStruct {
a: u32,
b: u8,
c: u32,
d: u32,
}
const SIZE: usize = std::mem::size_of::<MyStruct>();
let mut out = [MaybeUninit::uninit(); SIZE];
let first_align = std::mem::offset_of!(MyStruct, a);
let second_align = std::mem::offset_of!(MyStruct, b);
let third_align = std::mem::offset_of!(MyStruct, c);
let fourth_align = std::mem::offset_of!(MyStruct, d);
for (i, byte) in 1234u32.to_le_bytes().iter().enumerate() {
out[i + first_align] = MaybeUninit::new(*byte);
}
for (i, byte) in 12u8.to_le_bytes().iter().enumerate() {
out[i + second_align] = MaybeUninit::new(*byte);
}
for (i, byte) in 13u32.to_le_bytes().iter().enumerate() {
out[i + third_align] = MaybeUninit::new(*byte);
}
for (i, byte) in 14u32.to_le_bytes().iter().enumerate() {
out[i + fourth_align] = MaybeUninit::new(*byte);
}
let out = unsafe { std::mem::transmute_copy::<[MaybeUninit<u8>; SIZE], MyStruct>(&out) };
assert_eq!(out.a, 1234);
assert_eq!(out.b, 12);
assert_eq!(out.c, 13);
assert_eq!(out.d, 14);
}
#[test]
fn test_serialize_const_layout_struct_list() {
#[derive(Clone, Copy, Debug, PartialEq, SerializeConst)]
struct Struct {
a: u32,
b: u8,
c: u32,
d: u32,
}
impl Struct {
#[allow(dead_code)]
const fn equal(&self, other: &Struct) -> bool {
self.a == other.a && self.b == other.b && self.c == other.c && self.d == other.d
}
}
#[derive(Clone, Copy, Debug, PartialEq, SerializeConst)]
struct OtherStruct {
a: u32,
b: u8,
c: Struct,
d: u32,
}
impl OtherStruct {
#[allow(dead_code)]
const fn equal(&self, other: &OtherStruct) -> bool {
self.a == other.a && self.b == other.b && self.c.equal(&other.c) && self.d == other.d
}
}
const INNER_DATA: Struct = Struct {
a: 0x11111111,
b: 0x22,
c: 0x33333333,
d: 0x44444444,
};
const DATA: [OtherStruct; 3] = [
OtherStruct {
a: 0x11111111,
b: 0x22,
c: INNER_DATA,
d: 0x44444444,
},
OtherStruct {
a: 0x111111,
b: 0x23,
c: INNER_DATA,
d: 0x44444444,
},
OtherStruct {
a: 0x11111111,
b: 0x11,
c: INNER_DATA,
d: 0x44441144,
},
];
const _ASSERT: () = {
let mut buf = ConstVec::new();
buf = serialize_const(&DATA, buf);
let buf = buf.read();
let [first, second, third] = match deserialize_const!([OtherStruct; 3], buf) {
Some((_, data)) => data,
None => panic!("data mismatch"),
};
if !(first.equal(&DATA[0]) && second.equal(&DATA[1]) && third.equal(&DATA[2])) {
panic!("data mismatch");
}
};
const _ASSERT_2: () = {
let mut buf = ConstVec::new();
const DATA_AGAIN: [[OtherStruct; 3]; 3] = [DATA, DATA, DATA];
buf = serialize_const(&DATA_AGAIN, buf);
let buf = buf.read();
let [first, second, third] = match deserialize_const!([[OtherStruct; 3]; 3], buf) {
Some((_, data)) => data,
None => panic!("data mismatch"),
};
if !(first[0].equal(&DATA[0]) && first[1].equal(&DATA[1]) && first[2].equal(&DATA[2])) {
panic!("data mismatch");
}
if !(second[0].equal(&DATA[0]) && second[1].equal(&DATA[1]) && second[2].equal(&DATA[2])) {
panic!("data mismatch");
}
if !(third[0].equal(&DATA[0]) && third[1].equal(&DATA[1]) && third[2].equal(&DATA[2])) {
panic!("data mismatch");
}
};
let mut buf = ConstVec::new();
buf = serialize_const(&DATA, buf);
println!("{:?}", buf.as_ref());
let buf = buf.read();
let (_, data2) = deserialize_const!([OtherStruct; 3], buf).unwrap();
assert_eq!(DATA, data2);
}
#[test]
fn test_serialize_const_layout_struct() {
#[derive(Debug, PartialEq, SerializeConst)]
struct Struct {
a: u32,
b: u8,
c: u32,
d: u32,
}
#[derive(Debug, PartialEq, SerializeConst)]
struct OtherStruct(u32, u8, Struct, u32);
println!("{:?}", OtherStruct::MEMORY_LAYOUT);
let data = Struct {
a: 0x11111111,
b: 0x22,
c: 0x33333333,
d: 0x44444444,
};
let data = OtherStruct(0x11111111, 0x22, data, 0x44444444);
let mut buf = ConstVec::new();
buf = serialize_const(&data, buf);
println!("{:?}", buf.as_ref());
let buf = buf.read();
let (_, data2) = deserialize_const!(OtherStruct, buf).unwrap();
assert_eq!(data, data2);
}
+28
View File
@@ -0,0 +1,28 @@
use const_serialize::{deserialize_const, serialize_const, ConstVec};
#[test]
fn test_serialize_const_layout_tuple() {
let mut buf = ConstVec::new();
buf = serialize_const(&(1234u32, 5678u16), buf);
let buf = buf.read();
assert_eq!(
deserialize_const!((u32, u16), buf).unwrap().1,
(1234u32, 5678u16)
);
let mut buf = ConstVec::new();
buf = serialize_const(&(1234f64, 5678u16, 90u8), buf);
let buf = buf.read();
assert_eq!(
deserialize_const!((f64, u16, u8), buf).unwrap().1,
(1234f64, 5678u16, 90u8)
);
let mut buf = ConstVec::new();
buf = serialize_const(&(1234u32, 5678u16, 90u8, 1000000f64), buf);
let buf = buf.read();
assert_eq!(
deserialize_const!((u32, u16, u8, f64), buf).unwrap().1,
(1234u32, 5678u16, 90u8, 1000000f64)
);
}