swaits/dioxus
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Switch const serialize to use CBOR (#4932)

Browse Source 
* implement const cbor encoding + decoding for basic types

* implement map encoding and decoding

* fix map test

* all tests passing

* dynamically sized arrays

* smaller str encoding

* swap array and list and add more documentation

* move serialization and deserialization for each struct together

* support both legacy and new assets

* bump const serialize version

* fix asset conversion code

* pull out manganis symbol struct from tuble

* test reading old asset versions

* more list/array cleanup and bump rust version

* fix clippy

* switch cli opt port

* install playwright deps

* split cli-optimization package to avoid two dioxus' in the same crate

* fix clippy

* force the toast.png asset to be included in the linker stage #[used] has platform depedant behavior

* bundle both formats in the link section and only use the asset the cli changed

* remove sudo which doesn't exist on windows

* add binstall to the playwright workflow

* separate bin target for the old cli

* lockfile

* cargo update to get versions shaken out

* notes on binstalls, add back `.read()` for compat

* use 0.8.0-alpha.0 for const-serialize instead of 0.8

this ensures we can still publish breaking changes on 0.8 without taking over the full version

* fix clippy in example

* small debug nits

* update rust-overlay in nix

* fix flake?

* migrate to new structure

* just comment out binstalls

* just remove old test harness

---------

Co-authored-by: Jonathan Kelley <jkelleyrtp@gmail.com>
This commit is contained in:
Evan AlmloffandJonathan Kelley authored and GitHub committed 2025-12-28 21:49:34 -05:00
1 parent c68a47b2d3
commit c977cc125e
48 files changed
+3386 -2007
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

packages/const-serialize/src/array.rs
+64
View File
@@ -0,0 +1,64 @@
use crate::*;
/// The layout for a constant sized array. The array layout is just a length and an item layout.
#[derive(Debug, Copy, Clone)]
pub struct ArrayLayout {
pub(crate) len: usize,
pub(crate) item_layout: &'static Layout,
}
impl ArrayLayout {
/// Create a new array layout
pub const fn new(len: usize, item_layout: &'static Layout) -> Self {
Self { len, item_layout }
}
}
unsafe impl<const N: usize, T: SerializeConst> SerializeConst for [T; N] {
const MEMORY_LAYOUT: Layout = Layout::Array(ArrayLayout {
len: N,
item_layout: &T::MEMORY_LAYOUT,
});
}
/// Serialize a constant sized array that is stored at the pointer passed in
pub(crate) const unsafe fn serialize_const_array(
ptr: *const (),
mut to: ConstVec<u8>,
layout: &ArrayLayout,
) -> ConstVec<u8> {
let len = layout.len;
let mut i = 0;
to = write_array(to, len);
while i < len {
let field = ptr.wrapping_byte_offset((i * layout.item_layout.size()) as _);
to = serialize_const_ptr(field, to, layout.item_layout);
i += 1;
}
to
}
/// Deserialize an array type into the out buffer at the offset passed in. Returns a new version of the buffer with the data added.
pub(crate) const fn deserialize_const_array<'a>(
from: &'a [u8],
layout: &ArrayLayout,
mut out: &mut [MaybeUninit<u8>],
) -> Option<&'a [u8]> {
let item_layout = layout.item_layout;
let Ok((_, mut from)) = take_array(from) else {
return None;
};
let mut i = 0;
while i < layout.len {
let Some(new_from) = deserialize_const_ptr(from, item_layout, out) else {
return None;
};
let Some((_, item_out)) = out.split_at_mut_checked(item_layout.size()) else {
return None;
};
out = item_out;
from = new_from;
i += 1;
}
Some(from)
}
packages/const-serialize/src/cbor.rs
+597
View File
@@ -0,0 +1,597 @@
//! Const serialization utilities for the CBOR data format.
//!
//! ## Overview of the format
//!
//! Const serialize only supports a subset of the CBOR format, specifically the major types:
//! - UnsignedInteger
//! - NegativeInteger
//! - Bytes
//! - String
//! - Array
//!
//! Each item in CBOR starts with a leading byte, which determines the type of the item and additional information.
//! The additional information is encoded in the lower 5 bits of the leading byte and generally indicates either a
//! small number or how many of the next bytes are part of the first number.
//!
//! Resources:
//! The spec: <https://www.rfc-editor.org/rfc/rfc8949.html>
//! A playground to check examples against: <https://cbor.me/>
use crate::ConstVec;
/// Each item in CBOR starts with a leading byte, which determines the type of the item and additional information.
///
/// The first 3 bits of the leading byte are the major type, which indicates the type of the item.
#[repr(u8)]
#[derive(PartialEq)]
enum MajorType {
/// An unsigned integer in the range 0..2^64. The value of the number is encoded in the remaining bits of the leading byte and any additional bytes.
UnsignedInteger = 0,
/// An unsigned integer in the range -2^64..-1. The value of the number is encoded in the remaining bits of the leading byte and any additional bytes
NegativeInteger = 1,
/// A byte sequence. The number of bytes in the sequence is encoded in the remaining bits of the leading byte and any additional bytes.
Bytes = 2,
/// A text sequence. The number of bytes in the sequence is encoded in the remaining bits of the leading byte and any additional bytes.
Text = 3,
/// A dynamically sized array of non-uniform data items. The number of items in the array is encoded in the remaining bits of the leading byte and any additional bytes.
Array = 4,
/// A map of pairs of data items. The first item in each pair is the key and the second item is the value. The number of items in the array is encoded in the remaining bits of the leading byte and any additional bytes.
Map = 5,
/// Tagged values - not supported
Tagged = 6,
/// Floating point values - not supported
Float = 7,
}
impl MajorType {
/// The bitmask for the major type in the leading byte
const MASK: u8 = 0b0001_1111;
const fn from_byte(byte: u8) -> Self {
match byte >> 5 {
0 => MajorType::UnsignedInteger,
1 => MajorType::NegativeInteger,
2 => MajorType::Bytes,
3 => MajorType::Text,
4 => MajorType::Array,
5 => MajorType::Map,
6 => MajorType::Tagged,
7 => MajorType::Float,
_ => panic!("Invalid major type"),
}
}
}
/// Get the length of the item in bytes without deserialization.
const fn item_length(bytes: &[u8]) -> Result<usize, ()> {
let [head, rest @ ..] = bytes else {
return Err(());
};
let major = MajorType::from_byte(*head);
let additional_information = *head & MajorType::MASK;
let length_of_item = match major {
// The length of the number is the total of:
// - The length of the number (which may be 0 if the number is encoded in additional information)
MajorType::UnsignedInteger | MajorType::NegativeInteger => {
get_length_of_number(additional_information) as usize
}
// The length of the text or bytes is the total of:
// - The length of the number that denotes the length of the text or bytes
// - The length of the text or bytes themselves
MajorType::Text | MajorType::Bytes => {
let length_of_number = get_length_of_number(additional_information);
let Ok((length_of_bytes, _)) =
grab_u64_with_byte_length(rest, length_of_number, additional_information)
else {
return Err(());
};
length_of_number as usize + length_of_bytes as usize
}
// The length of the map is the total of:
// - The length of the number that denotes the number of items
// - The length of the pairs of items themselves
MajorType::Array | MajorType::Map => {
let length_of_number = get_length_of_number(additional_information);
let Ok((length_of_items, _)) =
grab_u64_with_byte_length(rest, length_of_number, additional_information)
else {
return Err(());
};
let mut total_length = length_of_number as usize;
let mut items_left = length_of_items * if let MajorType::Map = major { 2 } else { 1 };
while items_left > 0 {
let Some((_, after)) = rest.split_at_checked(total_length) else {
return Err(());
};
let Ok(item_length) = item_length(after) else {
return Err(());
};
total_length += item_length;
items_left -= 1;
}
total_length
}
_ => return Err(()),
};
let length_of_head = 1;
Ok(length_of_head + length_of_item)
}
/// Read a number from the buffer, returning the number and the remaining bytes.
pub(crate) const fn take_number(bytes: &[u8]) -> Result<(i64, &[u8]), ()> {
let [head, rest @ ..] = bytes else {
return Err(());
};
let major = MajorType::from_byte(*head);
let additional_information = *head & MajorType::MASK;
match major {
MajorType::UnsignedInteger => {
let Ok((number, rest)) = grab_u64(rest, additional_information) else {
return Err(());
};
Ok((number as i64, rest))
}
MajorType::NegativeInteger => {
let Ok((number, rest)) = grab_u64(rest, additional_information) else {
return Err(());
};
Ok((-(1 + number as i64), rest))
}
_ => Err(()),
}
}
/// Write a number to the buffer
pub(crate) const fn write_number<const MAX_SIZE: usize>(
vec: ConstVec<u8, MAX_SIZE>,
number: i64,
) -> ConstVec<u8, MAX_SIZE> {
match number {
0.. => write_major_type_and_u64(vec, MajorType::UnsignedInteger, number as u64),
..0 => write_major_type_and_u64(vec, MajorType::NegativeInteger, (-(number + 1)) as u64),
}
}
/// Write the major type along with a number to the buffer. The first byte
/// contains both the major type and the additional information which contains
/// either the number itself or the number of extra bytes the number occupies.
const fn write_major_type_and_u64<const MAX_SIZE: usize>(
vec: ConstVec<u8, MAX_SIZE>,
major: MajorType,
number: u64,
) -> ConstVec<u8, MAX_SIZE> {
let major = (major as u8) << 5;
match number {
// For numbers less than 24, store the number in the lower bits
// of the first byte
0..24 => {
let additional_information = number as u8;
let byte = major | additional_information;
vec.push(byte)
}
// For larger numbers, store the number of extra bytes the number occupies
24.. => {
let log2_additional_bytes = log2_bytes_for_number(number);
let additional_bytes = 1 << log2_additional_bytes;
let additional_information = log2_additional_bytes + 24;
let byte = major | additional_information;
let mut vec = vec.push(byte);
let mut byte = 0;
while byte < additional_bytes {
vec = vec.push((number >> ((additional_bytes - byte - 1) * 8)) as u8);
byte += 1;
}
vec
}
}
}
/// Find the number of bytes required to store a number and return the log2 of the number of bytes.
/// This is the number stored in the additional information field if the number is more than 24.
const fn log2_bytes_for_number(number: u64) -> u8 {
let required_bytes = ((64 - number.leading_zeros()).div_ceil(8)) as u8;
#[allow(clippy::match_overlapping_arm)]
match required_bytes {
..=1 => 0,
..=2 => 1,
..=4 => 2,
_ => 3,
}
}
/// Take bytes from a slice and return the bytes and the remaining slice.
pub(crate) const fn take_bytes(bytes: &[u8]) -> Result<(&[u8], &[u8]), ()> {
let [head, rest @ ..] = bytes else {
return Err(());
};
let major = MajorType::from_byte(*head);
let additional_information = *head & MajorType::MASK;
if let MajorType::Bytes = major {
take_bytes_from(rest, additional_information)
} else {
Err(())
}
}
/// Write bytes to a buffer and return the new buffer.
pub(crate) const fn write_bytes<const MAX_SIZE: usize>(
vec: ConstVec<u8, MAX_SIZE>,
bytes: &[u8],
) -> ConstVec<u8, MAX_SIZE> {
let vec = write_major_type_and_u64(vec, MajorType::Bytes, bytes.len() as u64);
vec.extend(bytes)
}
/// Take a string from a buffer and return the string and the remaining buffer.
pub(crate) const fn take_str(bytes: &[u8]) -> Result<(&str, &[u8]), ()> {
let [head, rest @ ..] = bytes else {
return Err(());
};
let major = MajorType::from_byte(*head);
let additional_information = *head & MajorType::MASK;
if let MajorType::Text = major {
let Ok((bytes, rest)) = take_bytes_from(rest, additional_information) else {
return Err(());
};
let Ok(string) = std::str::from_utf8(bytes) else {
return Err(());
};
Ok((string, rest))
} else {
Err(())
}
}
/// Write a string to a buffer and return the new buffer.
pub(crate) const fn write_str<const MAX_SIZE: usize>(
vec: ConstVec<u8, MAX_SIZE>,
string: &str,
) -> ConstVec<u8, MAX_SIZE> {
let vec = write_major_type_and_u64(vec, MajorType::Text, string.len() as u64);
vec.extend(string.as_bytes())
}
/// Take the length and header of an array from a buffer and return the length and the remaining buffer.
/// You must loop over the elements of the array and parse them outside of this method.
pub(crate) const fn take_array(bytes: &[u8]) -> Result<(usize, &[u8]), ()> {
let [head, rest @ ..] = bytes else {
return Err(());
};
let major = MajorType::from_byte(*head);
let additional_information = *head & MajorType::MASK;
if let MajorType::Array = major {
let Ok((length, rest)) = take_len_from(rest, additional_information) else {
return Err(());
};
Ok((length as usize, rest))
} else {
Err(())
}
}
/// Write the header and length of an array.
pub(crate) const fn write_array<const MAX_SIZE: usize>(
vec: ConstVec<u8, MAX_SIZE>,
len: usize,
) -> ConstVec<u8, MAX_SIZE> {
write_major_type_and_u64(vec, MajorType::Array, len as u64)
}
/// Write the header and length of a map.
pub(crate) const fn write_map<const MAX_SIZE: usize>(
vec: ConstVec<u8, MAX_SIZE>,
len: usize,
) -> ConstVec<u8, MAX_SIZE> {
// We write 2 * len as the length of the map because each key-value pair is a separate entry.
write_major_type_and_u64(vec, MajorType::Map, len as u64)
}
/// Write the key of a map entry.
pub(crate) const fn write_map_key<const MAX_SIZE: usize>(
value: ConstVec<u8, MAX_SIZE>,
key: &str,
) -> ConstVec<u8, MAX_SIZE> {
write_str(value, key)
}
/// Take a map from the byte slice and return the map reference and the remaining bytes.
pub(crate) const fn take_map<'a>(bytes: &'a [u8]) -> Result<(MapRef<'a>, &'a [u8]), ()> {
let [head, rest @ ..] = bytes else {
return Err(());
};
let major = MajorType::from_byte(*head);
let additional_information = *head & MajorType::MASK;
if let MajorType::Map = major {
let Ok((length, rest)) = take_len_from(rest, additional_information) else {
return Err(());
};
let mut after_map = rest;
let mut items_left = length * 2;
while items_left > 0 {
// Skip the value
let Ok(len) = item_length(after_map) else {
return Err(());
};
let Some((_, rest)) = after_map.split_at_checked(len) else {
return Err(());
};
after_map = rest;
items_left -= 1;
}
Ok((MapRef::new(rest, length as usize), after_map))
} else {
Err(())
}
}
/// A reference to a CBOR map.
pub(crate) struct MapRef<'a> {
/// The bytes of the map.
pub(crate) bytes: &'a [u8],
/// The length of the map.
pub(crate) len: usize,
}
impl<'a> MapRef<'a> {
/// Create a new map reference.
const fn new(bytes: &'a [u8], len: usize) -> Self {
Self { bytes, len }
}
/// Find a key in the map and return the buffer associated with it.
pub(crate) const fn find(&self, key: &str) -> Result<Option<&[u8]>, ()> {
let mut bytes = self.bytes;
let mut items_left = self.len;
while items_left > 0 {
let Ok((str, rest)) = take_str(bytes) else {
return Err(());
};
if str_eq(key, str) {
return Ok(Some(rest));
}
// Skip the value associated with the key we don't care about
let Ok(len) = item_length(rest) else {
return Err(());
};
let Some((_, rest)) = rest.split_at_checked(len) else {
return Err(());
};
bytes = rest;
items_left -= 1;
}
Ok(None)
}
}
/// Compare two strings for equality at compile time.
pub(crate) const fn str_eq(a: &str, b: &str) -> bool {
let a_bytes = a.as_bytes();
let b_bytes = b.as_bytes();
let a_len = a_bytes.len();
let b_len = b_bytes.len();
if a_len != b_len {
return false;
}
let mut index = 0;
while index < a_len {
if a_bytes[index] != b_bytes[index] {
return false;
}
index += 1;
}
true
}
/// Take the length from the additional information byte and return it along with the remaining bytes.
const fn take_len_from(rest: &[u8], additional_information: u8) -> Result<(u64, &[u8]), ()> {
match additional_information {
// If additional_information < 24, the argument's value is the value of the additional information.
0..24 => Ok((additional_information as u64, rest)),
// If additional_information is between 24 and 28, the argument's value is held in the n following bytes.
24..28 => {
let Ok((number, rest)) = grab_u64(rest, additional_information) else {
return Err(());
};
Ok((number, rest))
}
_ => Err(()),
}
}
/// Take a list of bytes from the byte slice and the additional information byte
/// and return the bytes and the remaining bytes.
pub(crate) const fn take_bytes_from(
rest: &[u8],
additional_information: u8,
) -> Result<(&[u8], &[u8]), ()> {
let Ok((number, rest)) = grab_u64(rest, additional_information) else {
return Err(());
};
let Some((bytes, rest)) = rest.split_at_checked(number as usize) else {
return Err(());
};
Ok((bytes, rest))
}
/// Find the length of the number based on the additional information byte.
const fn get_length_of_number(additional_information: u8) -> u8 {
match additional_information {
0..24 => 0,
24..28 => 1 << (additional_information - 24),
_ => 0,
}
}
/// Read a u64 from the byte slice and the additional information byte.
const fn grab_u64(rest: &[u8], additional_information: u8) -> Result<(u64, &[u8]), ()> {
grab_u64_with_byte_length(
rest,
get_length_of_number(additional_information),
additional_information,
)
}
/// Read a u64 from the byte slice and the additional information byte along with the byte length.
const fn grab_u64_with_byte_length(
mut rest: &[u8],
byte_length: u8,
additional_information: u8,
) -> Result<(u64, &[u8]), ()> {
match byte_length {
0 => Ok((additional_information as u64, rest)),
n => {
let mut value = 0;
let mut count = 0;
while count < n {
let [next, remaining @ ..] = rest else {
return Err(());
};
value = (value << 8) | *next as u64;
rest = remaining;
count += 1;
}
Ok((value, rest))
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parse_byte() {
for byte in 0..=255 {
let bytes = if byte < 24 { [byte, 0] } else { [24, byte] };
let (item, _) = take_number(&bytes).unwrap();
assert_eq!(item, byte as _);
}
for byte in 1..=255 {
let bytes = if byte < 24 {
[(byte - 1) | 0b0010_0000, 0]
} else {
[0b0010_0000 | 24, byte - 1]
};
let (item, _) = take_number(&bytes).unwrap();
assert_eq!(item, -(byte as i64));
}
}
#[test]
fn test_byte_roundtrip() {
for byte in 0..=255 {
let vec = write_number(ConstVec::new(), byte as _);
println!("{vec:?}");
let (item, _) = take_number(vec.as_ref()).unwrap();
assert_eq!(item, byte as _);
}
for byte in 0..=255 {
let vec = write_number(ConstVec::new(), -(byte as i64));
let (item, _) = take_number(vec.as_ref()).unwrap();
assert_eq!(item, -(byte as i64));
}
}
#[test]
fn test_number_roundtrip() {
for _ in 0..100 {
let value = rand::random::<i64>();
let vec = write_number(ConstVec::new(), value);
let (item, _) = take_number(vec.as_ref()).unwrap();
assert_eq!(item, value);
}
}
#[test]
fn test_bytes_roundtrip() {
for _ in 0..100 {
let len = (rand::random::<u8>() % 100) as usize;
let bytes = rand::random::<[u8; 100]>();
let vec = write_bytes(ConstVec::new(), &bytes[..len]);
let (item, _) = take_bytes(vec.as_ref()).unwrap();
assert_eq!(item, &bytes[..len]);
}
}
#[test]
fn test_array_roundtrip() {
for _ in 0..100 {
let len = (rand::random::<u8>() % 100) as usize;
let mut vec = write_array(ConstVec::new(), len);
for i in 0..len {
vec = write_number(vec, i as _);
}
let (len, mut remaining) = take_array(vec.as_ref()).unwrap();
for i in 0..len {
let (item, rest) = take_number(remaining).unwrap();
remaining = rest;
assert_eq!(item, i as i64);
}
}
}
#[test]
fn test_map_roundtrip() {
use rand::prelude::SliceRandom;
for _ in 0..100 {
let len = (rand::random::<u8>() % 10) as usize;
let mut vec = write_map(ConstVec::new(), len);
let mut random_order_indexes = (0..len).collect::<Vec<_>>();
random_order_indexes.shuffle(&mut rand::rng());
for &i in &random_order_indexes {
vec = write_map_key(vec, &i.to_string());
vec = write_number(vec, i as _);
}
println!("len: {}", len);
println!("Map: {:?}", vec);
let (map, remaining) = take_map(vec.as_ref()).unwrap();
println!("remaining: {:?}", remaining);
assert!(remaining.is_empty());
for i in 0..len {
let key = i.to_string();
let key_location = map
.find(&key)
.expect("encoding is valid")
.expect("key exists");
let (value, _) = take_number(key_location).unwrap();
assert_eq!(value, i as i64);
}
}
}
#[test]
fn test_item_length_str() {
#[rustfmt::skip]
let input = [
/* text(1) */ 0x61,
/* "1" */ 0x31,
/* text(1) */ 0x61,
/* "1" */ 0x31,
];
let Ok(length) = item_length(&input) else {
panic!("Failed to calculate length");
};
assert_eq!(length, 2);
}
#[test]
fn test_item_length_map() {
#[rustfmt::skip]
let input = [
/* map(1) */ 0xA1,
/* text(1) */ 0x61,
/* "A" */ 0x41,
/* map(2) */ 0xA2,
/* text(3) */ 0x63,
/* "one" */ 0x6F, 0x6E, 0x65,
/* unsigned(286331153) */ 0x1A, 0x11, 0x11, 0x11, 0x11,
/* text(3) */ 0x63,
/* "two" */ 0x74, 0x77, 0x6F,
/* unsigned(34) */ 0x18, 0x22,
];
let Ok(length) = item_length(&input) else {
panic!("Failed to calculate length");
};
assert_eq!(length, input.len());
}
}
packages/const-serialize/src/enum.rs
+135
View File
@@ -0,0 +1,135 @@
use crate::*;
/// Serialize an enum that is stored at the pointer passed in
pub(crate) const unsafe fn serialize_const_enum(
ptr: *const (),
mut to: ConstVec<u8>,
layout: &EnumLayout,
) -> ConstVec<u8> {
let byte_ptr = ptr as *const u8;
let discriminant = layout.discriminant.read(byte_ptr);
let mut i = 0;
while i < layout.variants.len() {
// If the variant is the discriminated one, serialize it
let EnumVariant {
tag, name, data, ..
} = &layout.variants[i];
if discriminant == *tag {
to = write_map(to, 1);
to = write_map_key(to, name);
let data_ptr = ptr.wrapping_byte_offset(layout.variants_offset as _);
to = serialize_const_struct(data_ptr, to, data);
break;
}
i += 1;
}
to
}
/// The layout for an enum. The enum layout is just a discriminate size and a tag layout.
#[derive(Debug, Copy, Clone)]
pub struct EnumLayout {
pub(crate) size: usize,
discriminant: PrimitiveLayout,
variants_offset: usize,
variants: &'static [EnumVariant],
}
impl EnumLayout {
/// Create a new enum layout
pub const fn new(
size: usize,
discriminant: PrimitiveLayout,
variants: &'static [EnumVariant],
) -> Self {
let mut max_align = 1;
let mut i = 0;
while i < variants.len() {
let EnumVariant { align, .. } = &variants[i];
if *align > max_align {
max_align = *align;
}
i += 1;
}
let variants_offset_raw = discriminant.size;
let padding = (max_align - (variants_offset_raw % max_align)) % max_align;
let variants_offset = variants_offset_raw + padding;
assert!(variants_offset % max_align == 0);
Self {
size,
discriminant,
variants_offset,
variants,
}
}
}
/// The layout for an enum variant. The enum variant layout is just a struct layout with a tag and alignment.
#[derive(Debug, Copy, Clone)]
pub struct EnumVariant {
name: &'static str,
// Note: tags may not be sequential
tag: u32,
data: StructLayout,
align: usize,
}
impl EnumVariant {
/// Create a new enum variant layout
pub const fn new(name: &'static str, tag: u32, data: StructLayout, align: usize) -> Self {
Self {
name,
tag,
data,
align,
}
}
}
/// Deserialize an enum type into the out buffer at the offset passed in. Returns a new version of the buffer with the data added.
pub(crate) const fn deserialize_const_enum<'a>(
from: &'a [u8],
layout: &EnumLayout,
out: &mut [MaybeUninit<u8>],
) -> Option<&'a [u8]> {
// First, deserialize the map
let Ok((map, remaining)) = take_map(from) else {
return None;
};
// Then get the only field which is the tag
let Ok((deserilized_name, from)) = take_str(map.bytes) else {
return None;
};
// Then, deserialize the variant
let mut i = 0;
let mut matched_variant = false;
while i < layout.variants.len() {
// If the variant is the discriminated one, deserialize it
let EnumVariant {
name, data, tag, ..
} = &layout.variants[i];
if str_eq(deserilized_name, name) {
layout.discriminant.write(*tag, out);
let Some((_, out)) = out.split_at_mut_checked(layout.variants_offset) else {
return None;
};
if deserialize_const_struct(from, data, out).is_none() {
return None;
}
matched_variant = true;
break;
}
i += 1;
}
if !matched_variant {
return None;
}
Some(remaining)
}
packages/const-serialize/src/lib.rs
+47 -781
View File
@@ -1,126 +1,32 @@
#![doc = include_str!("../README.md")]
#![warn(missing_docs)]
use std::{char, mem::MaybeUninit};
use std::mem::MaybeUninit;
mod const_buffers;
mod const_vec;
pub use const_buffers::ConstReadBuffer;
mod cbor;
mod const_vec;
mod r#enum;
pub use r#enum::*;
mod r#struct;
pub use r#struct::*;
mod primitive;
pub use primitive::*;
mod list;
pub use list::*;
mod array;
pub use array::*;
mod str;
pub use str::*;
pub use const_serialize_macro::SerializeConst;
pub use const_vec::ConstVec;
/// Plain old data for a field. Stores the offset of the field in the struct and the layout of the field.
#[derive(Debug, Copy, Clone)]
pub struct StructFieldLayout {
offset: usize,
layout: Layout,
}
impl StructFieldLayout {
/// Create a new struct field layout
pub const fn new(offset: usize, layout: Layout) -> Self {
Self { offset, layout }
}
}
/// Layout for a struct. The struct layout is just a list of fields with offsets
#[derive(Debug, Copy, Clone)]
pub struct StructLayout {
size: usize,
data: &'static [StructFieldLayout],
}
impl StructLayout {
/// Create a new struct layout
pub const fn new(size: usize, data: &'static [StructFieldLayout]) -> Self {
Self { size, data }
}
}
/// The layout for an enum. The enum layout is just a discriminate size and a tag layout.
#[derive(Debug, Copy, Clone)]
pub struct EnumLayout {
size: usize,
discriminant: PrimitiveLayout,
variants_offset: usize,
variants: &'static [EnumVariant],
}
impl EnumLayout {
/// Create a new enum layout
pub const fn new(
size: usize,
discriminant: PrimitiveLayout,
variants: &'static [EnumVariant],
) -> Self {
let mut max_align = 1;
let mut i = 0;
while i < variants.len() {
let EnumVariant { align, .. } = &variants[i];
if *align > max_align {
max_align = *align;
}
i += 1;
}
let variants_offset_raw = discriminant.size;
let padding = (max_align - (variants_offset_raw % max_align)) % max_align;
let variants_offset = variants_offset_raw + padding;
assert!(variants_offset % max_align == 0);
Self {
size,
discriminant,
variants_offset,
variants,
}
}
}
/// The layout for an enum variant. The enum variant layout is just a struct layout with a tag and alignment.
#[derive(Debug, Copy, Clone)]
pub struct EnumVariant {
// Note: tags may not be sequential
tag: u32,
data: StructLayout,
align: usize,
}
impl EnumVariant {
/// Create a new enum variant layout
pub const fn new(tag: u32, data: StructLayout, align: usize) -> Self {
Self { tag, data, align }
}
}
/// The layout for a constant sized array. The array layout is just a length and an item layout.
#[derive(Debug, Copy, Clone)]
pub struct ListLayout {
len: usize,
item_layout: &'static Layout,
}
impl ListLayout {
/// Create a new list layout
pub const fn new(len: usize, item_layout: &'static Layout) -> Self {
Self { len, item_layout }
}
}
/// The layout for a primitive type. The bytes will be reversed if the target is big endian.
#[derive(Debug, Copy, Clone)]
pub struct PrimitiveLayout {
size: usize,
}
impl PrimitiveLayout {
/// Create a new primitive layout
pub const fn new(size: usize) -> Self {
Self { size }
}
}
use crate::cbor::{
str_eq, take_array, take_bytes, take_map, take_number, take_str, write_array, write_bytes,
write_map, write_map_key, write_number,
};
/// The layout for a type. This layout defines a sequence of locations and reversed or not bytes. These bytes will be copied from during serialization and copied into during deserialization.
#[derive(Debug, Copy, Clone)]
@@ -129,10 +35,12 @@ pub enum Layout {
Enum(EnumLayout),
/// A struct layout
Struct(StructLayout),
/// A list layout
List(ListLayout),
/// An array layout
Array(ArrayLayout),
/// A primitive layout
Primitive(PrimitiveLayout),
/// A dynamically sized list layout
List(ListLayout),
}
impl Layout {
@@ -141,7 +49,8 @@ impl Layout {
match self {
Layout::Enum(layout) => layout.size,
Layout::Struct(layout) => layout.size,
Layout::List(layout) => layout.len * layout.item_layout.size(),
Layout::Array(layout) => layout.len * layout.item_layout.size(),
Layout::List(layout) => layout.size,
Layout::Primitive(layout) => layout.size,
}
}
@@ -158,533 +67,16 @@ pub unsafe trait SerializeConst: Sized {
const _ASSERT: () = assert!(Self::MEMORY_LAYOUT.size() == std::mem::size_of::<Self>());
}
macro_rules! impl_serialize_const {
($type:ty) => {
unsafe impl SerializeConst for $type {
const MEMORY_LAYOUT: Layout = Layout::Primitive(PrimitiveLayout {
size: std::mem::size_of::<$type>(),
});
}
};
}
impl_serialize_const!(u8);
impl_serialize_const!(u16);
impl_serialize_const!(u32);
impl_serialize_const!(u64);
impl_serialize_const!(i8);
impl_serialize_const!(i16);
impl_serialize_const!(i32);
impl_serialize_const!(i64);
impl_serialize_const!(bool);
impl_serialize_const!(f32);
impl_serialize_const!(f64);
unsafe impl<const N: usize, T: SerializeConst> SerializeConst for [T; N] {
const MEMORY_LAYOUT: Layout = Layout::List(ListLayout {
len: N,
item_layout: &T::MEMORY_LAYOUT,
});
}
macro_rules! impl_serialize_const_tuple {
($($generic:ident: $generic_number:expr),*) => {
impl_serialize_const_tuple!(@impl ($($generic,)*) = $($generic: $generic_number),*);
};
(@impl $inner:ty = $($generic:ident: $generic_number:expr),*) => {
unsafe impl<$($generic: SerializeConst),*> SerializeConst for ($($generic,)*) {
const MEMORY_LAYOUT: Layout = {
Layout::Struct(StructLayout {
size: std::mem::size_of::<($($generic,)*)>(),
data: &[
$(
StructFieldLayout::new(std::mem::offset_of!($inner, $generic_number), $generic::MEMORY_LAYOUT),
)*
],
})
};
}
};
}
impl_serialize_const_tuple!(T1: 0);
impl_serialize_const_tuple!(T1: 0, T2: 1);
impl_serialize_const_tuple!(T1: 0, T2: 1, T3: 2);
impl_serialize_const_tuple!(T1: 0, T2: 1, T3: 2, T4: 3);
impl_serialize_const_tuple!(T1: 0, T2: 1, T3: 2, T4: 3, T5: 4);
impl_serialize_const_tuple!(T1: 0, T2: 1, T3: 2, T4: 3, T5: 4, T6: 5);
impl_serialize_const_tuple!(T1: 0, T2: 1, T3: 2, T4: 3, T5: 4, T6: 5, T7: 6);
impl_serialize_const_tuple!(T1: 0, T2: 1, T3: 2, T4: 3, T5: 4, T6: 5, T7: 6, T8: 7);
impl_serialize_const_tuple!(T1: 0, T2: 1, T3: 2, T4: 3, T5: 4, T6: 5, T7: 6, T8: 7, T9: 8);
impl_serialize_const_tuple!(T1: 0, T2: 1, T3: 2, T4: 3, T5: 4, T6: 5, T7: 6, T8: 7, T9: 8, T10: 9);
const MAX_STR_SIZE: usize = 256;
/// A string that is stored in a constant sized buffer that can be serialized and deserialized at compile time
#[derive(Eq, PartialEq, PartialOrd, Clone, Copy, Hash)]
pub struct ConstStr {
bytes: [u8; MAX_STR_SIZE],
len: u32,
}
#[cfg(feature = "serde")]
mod serde_bytes {
use serde::{Deserialize, Serialize, Serializer};
use crate::ConstStr;
impl Serialize for ConstStr {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_str(self.as_str())
}
}
impl<'de> Deserialize<'de> for ConstStr {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
let s = String::deserialize(deserializer)?;
Ok(ConstStr::new(&s))
}
}
}
unsafe impl SerializeConst for ConstStr {
const MEMORY_LAYOUT: Layout = Layout::Struct(StructLayout {
size: std::mem::size_of::<Self>(),
data: &[
StructFieldLayout::new(
std::mem::offset_of!(Self, bytes),
Layout::List(ListLayout {
len: MAX_STR_SIZE,
item_layout: &Layout::Primitive(PrimitiveLayout {
size: std::mem::size_of::<u8>(),
}),
}),
),
StructFieldLayout::new(
std::mem::offset_of!(Self, len),
Layout::Primitive(PrimitiveLayout {
size: std::mem::size_of::<u32>(),
}),
),
],
});
}
impl ConstStr {
/// Create a new constant string
pub const fn new(s: &str) -> Self {
let str_bytes = s.as_bytes();
let mut bytes = [0; MAX_STR_SIZE];
let mut i = 0;
while i < str_bytes.len() {
bytes[i] = str_bytes[i];
i += 1;
}
Self {
bytes,
len: str_bytes.len() as u32,
}
}
/// Get a reference to the string
pub const fn as_str(&self) -> &str {
let str_bytes = self.bytes.split_at(self.len as usize).0;
match std::str::from_utf8(str_bytes) {
Ok(s) => s,
Err(_) => panic!(
"Invalid utf8; ConstStr should only ever be constructed from valid utf8 strings"
),
}
}
/// Get the length of the string
pub const fn len(&self) -> usize {
self.len as usize
}
/// Check if the string is empty
pub const fn is_empty(&self) -> bool {
self.len == 0
}
/// Push a character onto the string
pub const fn push(self, byte: char) -> Self {
assert!(byte.is_ascii(), "Only ASCII bytes are supported");
let (bytes, len) = char_to_bytes(byte);
let (str, _) = bytes.split_at(len);
let Ok(str) = std::str::from_utf8(str) else {
panic!("Invalid utf8; char_to_bytes should always return valid utf8 bytes")
};
self.push_str(str)
}
/// Push a str onto the string
pub const fn push_str(self, str: &str) -> Self {
let Self { mut bytes, len } = self;
assert!(
str.len() + len as usize <= MAX_STR_SIZE,
"String is too long"
);
let str_bytes = str.as_bytes();
let new_len = len as usize + str_bytes.len();
let mut i = 0;
while i < str_bytes.len() {
bytes[len as usize + i] = str_bytes[i];
i += 1;
}
Self {
bytes,
len: new_len as u32,
}
}
/// Split the string at a byte index. The byte index must be a char boundary
pub const fn split_at(self, index: usize) -> (Self, Self) {
let (left, right) = self.bytes.split_at(index);
let left = match std::str::from_utf8(left) {
Ok(s) => s,
Err(_) => {
panic!("Invalid utf8; you cannot split at a byte that is not a char boundary")
}
};
let right = match std::str::from_utf8(right) {
Ok(s) => s,
Err(_) => {
panic!("Invalid utf8; you cannot split at a byte that is not a char boundary")
}
};
(Self::new(left), Self::new(right))
}
/// Split the string at the last occurrence of a character
pub const fn rsplit_once(&self, char: char) -> Option<(Self, Self)> {
let str = self.as_str();
let mut index = str.len() - 1;
// First find the bytes we are searching for
let (char_bytes, len) = char_to_bytes(char);
let (char_bytes, _) = char_bytes.split_at(len);
let bytes = str.as_bytes();
// Then walk backwards from the end of the string
loop {
let byte = bytes[index];
// Look for char boundaries in the string and check if the bytes match
if let Some(char_boundary_len) = utf8_char_boundary_to_char_len(byte) {
// Split up the string into three sections: [before_char, in_char, after_char]
let (before_char, after_index) = bytes.split_at(index);
let (in_char, after_char) = after_index.split_at(char_boundary_len as usize);
if in_char.len() != char_boundary_len as usize {
panic!("in_char.len() should always be equal to char_boundary_len as usize")
}
// Check if the bytes for the current char and the target char match
let mut in_char_eq = true;
let mut i = 0;
let min_len = if in_char.len() < char_bytes.len() {
in_char.len()
} else {
char_bytes.len()
};
while i < min_len {
in_char_eq &= in_char[i] == char_bytes[i];
i += 1;
}
// If they do, convert the bytes to strings and return the split strings
if in_char_eq {
let Ok(before_char_str) = std::str::from_utf8(before_char) else {
panic!("Invalid utf8; utf8_char_boundary_to_char_len should only return Some when the byte is a character boundary")
};
let Ok(after_char_str) = std::str::from_utf8(after_char) else {
panic!("Invalid utf8; utf8_char_boundary_to_char_len should only return Some when the byte is a character boundary")
};
return Some((Self::new(before_char_str), Self::new(after_char_str)));
}
}
match index.checked_sub(1) {
Some(new_index) => index = new_index,
None => return None,
}
}
}
/// Split the string at the first occurrence of a character
pub const fn split_once(&self, char: char) -> Option<(Self, Self)> {
let str = self.as_str();
let mut index = 0;
// First find the bytes we are searching for
let (char_bytes, len) = char_to_bytes(char);
let (char_bytes, _) = char_bytes.split_at(len);
let bytes = str.as_bytes();
// Then walk forwards from the start of the string
while index < bytes.len() {
let byte = bytes[index];
// Look for char boundaries in the string and check if the bytes match
if let Some(char_boundary_len) = utf8_char_boundary_to_char_len(byte) {
// Split up the string into three sections: [before_char, in_char, after_char]
let (before_char, after_index) = bytes.split_at(index);
let (in_char, after_char) = after_index.split_at(char_boundary_len as usize);
if in_char.len() != char_boundary_len as usize {
panic!("in_char.len() should always be equal to char_boundary_len as usize")
}
// Check if the bytes for the current char and the target char match
let mut in_char_eq = true;
let mut i = 0;
let min_len = if in_char.len() < char_bytes.len() {
in_char.len()
} else {
char_bytes.len()
};
while i < min_len {
in_char_eq &= in_char[i] == char_bytes[i];
i += 1;
}
// If they do, convert the bytes to strings and return the split strings
if in_char_eq {
let Ok(before_char_str) = std::str::from_utf8(before_char) else {
panic!("Invalid utf8; utf8_char_boundary_to_char_len should only return Some when the byte is a character boundary")
};
let Ok(after_char_str) = std::str::from_utf8(after_char) else {
panic!("Invalid utf8; utf8_char_boundary_to_char_len should only return Some when the byte is a character boundary")
};
return Some((Self::new(before_char_str), Self::new(after_char_str)));
}
}
index += 1
}
None
}
}
impl std::fmt::Debug for ConstStr {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{:?}", self.as_str())
}
}
#[test]
fn test_rsplit_once() {
let str = ConstStr::new("hello world");
assert_eq!(
str.rsplit_once(' '),
Some((ConstStr::new("hello"), ConstStr::new("world")))
);
let unicode_str = ConstStr::new("hi😀hello😀world😀world");
assert_eq!(
unicode_str.rsplit_once('😀'),
Some((ConstStr::new("hi😀hello😀world"), ConstStr::new("world")))
);
assert_eq!(unicode_str.rsplit_once('❌'), None);
for _ in 0..100 {
let random_str: String = (0..rand::random::<u8>() % 50)
.map(|_| rand::random::<char>())
.collect();
let konst = ConstStr::new(&random_str);
let mut seen_chars = std::collections::HashSet::new();
for char in random_str.chars().rev() {
let (char_bytes, len) = char_to_bytes(char);
let char_bytes = &char_bytes[..len];
assert_eq!(char_bytes, char.to_string().as_bytes());
if seen_chars.contains(&char) {
continue;
}
seen_chars.insert(char);
let (correct_left, correct_right) = random_str.rsplit_once(char).unwrap();
let (left, right) = konst.rsplit_once(char).unwrap();
println!("splitting {random_str:?} at {char:?}");
assert_eq!(left.as_str(), correct_left);
assert_eq!(right.as_str(), correct_right);
}
}
}
const CONTINUED_CHAR_MASK: u8 = 0b10000000;
const BYTE_CHAR_BOUNDARIES: [u8; 4] = [0b00000000, 0b11000000, 0b11100000, 0b11110000];
// Const version of https://doc.rust-lang.org/src/core/char/methods.rs.html#1765-1797
const fn char_to_bytes(char: char) -> ([u8; 4], usize) {
let code = char as u32;
let len = char.len_utf8();
let mut bytes = [0; 4];
match len {
1 => {
bytes[0] = code as u8;
}
2 => {
bytes[0] = ((code >> 6) & 0x1F) as u8 | BYTE_CHAR_BOUNDARIES[1];
bytes[1] = (code & 0x3F) as u8 | CONTINUED_CHAR_MASK;
}
3 => {
bytes[0] = ((code >> 12) & 0x0F) as u8 | BYTE_CHAR_BOUNDARIES[2];
bytes[1] = ((code >> 6) & 0x3F) as u8 | CONTINUED_CHAR_MASK;
bytes[2] = (code & 0x3F) as u8 | CONTINUED_CHAR_MASK;
}
4 => {
bytes[0] = ((code >> 18) & 0x07) as u8 | BYTE_CHAR_BOUNDARIES[3];
bytes[1] = ((code >> 12) & 0x3F) as u8 | CONTINUED_CHAR_MASK;
bytes[2] = ((code >> 6) & 0x3F) as u8 | CONTINUED_CHAR_MASK;
bytes[3] = (code & 0x3F) as u8 | CONTINUED_CHAR_MASK;
}
_ => panic!(
"encode_utf8: need more than 4 bytes to encode the unicode character, but the buffer has 4 bytes"
),
};
(bytes, len)
}
#[test]
fn fuzz_char_to_bytes() {
use std::char;
for _ in 0..100 {
let char = rand::random::<char>();
let (bytes, len) = char_to_bytes(char);
let str = std::str::from_utf8(&bytes[..len]).unwrap();
assert_eq!(char.to_string(), str);
}
}
const fn utf8_char_boundary_to_char_len(byte: u8) -> Option<u8> {
match byte {
0b00000000..=0b01111111 => Some(1),
0b11000000..=0b11011111 => Some(2),
0b11100000..=0b11101111 => Some(3),
0b11110000..=0b11111111 => Some(4),
_ => None,
}
}
#[test]
fn fuzz_utf8_byte_to_char_len() {
for _ in 0..100 {
let random_string: String = (0..rand::random::<u8>())
.map(|_| rand::random::<char>())
.collect();
let bytes = random_string.as_bytes();
let chars: std::collections::HashMap<_, _> = random_string.char_indices().collect();
for (i, byte) in bytes.iter().enumerate() {
match utf8_char_boundary_to_char_len(*byte) {
Some(char_len) => {
let char = chars
.get(&i)
.unwrap_or_else(|| panic!("{byte:b} is not a character boundary"));
assert_eq!(char.len_utf8(), char_len as usize);
}
None => {
assert!(!chars.contains_key(&i), "{byte:b} is a character boundary");
}
}
}
}
}
/// Serialize a struct that is stored at the pointer passed in
const fn serialize_const_struct(
ptr: *const (),
mut to: ConstVec<u8>,
layout: &StructLayout,
) -> ConstVec<u8> {
let mut i = 0;
while i < layout.data.len() {
// Serialize the field at the offset pointer in the struct
let StructFieldLayout { offset, layout } = &layout.data[i];
let field = ptr.wrapping_byte_add(*offset as _);
to = serialize_const_ptr(field, to, layout);
i += 1;
}
to
}
/// Serialize an enum that is stored at the pointer passed in
const fn serialize_const_enum(
ptr: *const (),
mut to: ConstVec<u8>,
layout: &EnumLayout,
) -> ConstVec<u8> {
let mut discriminant = 0;
let byte_ptr = ptr as *const u8;
let mut offset = 0;
while offset < layout.discriminant.size {
// If the bytes are reversed, walk backwards from the end of the number when pushing bytes
let byte = if cfg!(target_endian = "big") {
unsafe {
byte_ptr
.wrapping_byte_add((layout.discriminant.size - offset - 1) as _)
.read()
}
} else {
unsafe { byte_ptr.wrapping_byte_add(offset as _).read() }
};
to = to.push(byte);
discriminant |= (byte as u32) << (offset * 8);
offset += 1;
}
let mut i = 0;
while i < layout.variants.len() {
// If the variant is the discriminated one, serialize it
let EnumVariant { tag, data, .. } = &layout.variants[i];
if discriminant == *tag {
let data_ptr = ptr.wrapping_byte_offset(layout.variants_offset as _);
to = serialize_const_struct(data_ptr, to, data);
break;
}
i += 1;
}
to
}
/// Serialize a primitive type that is stored at the pointer passed in
const fn serialize_const_primitive(
ptr: *const (),
mut to: ConstVec<u8>,
layout: &PrimitiveLayout,
) -> ConstVec<u8> {
let ptr = ptr as *const u8;
let mut offset = 0;
while offset < layout.size {
// If the bytes are reversed, walk backwards from the end of the number when pushing bytes
if cfg!(any(target_endian = "big", feature = "test-big-endian")) {
to = to.push(unsafe {
ptr.wrapping_byte_offset((layout.size - offset - 1) as _)
.read()
});
} else {
to = to.push(unsafe { ptr.wrapping_byte_offset(offset as _).read() });
}
offset += 1;
}
to
}
/// Serialize a constant sized array that is stored at the pointer passed in
const fn serialize_const_list(
ptr: *const (),
mut to: ConstVec<u8>,
layout: &ListLayout,
) -> ConstVec<u8> {
let len = layout.len;
let mut i = 0;
while i < len {
let field = ptr.wrapping_byte_offset((i * layout.item_layout.size()) as _);
to = serialize_const_ptr(field, to, layout.item_layout);
i += 1;
}
to
}
/// Serialize a pointer to a type that is stored at the pointer passed in
const fn serialize_const_ptr(ptr: *const (), to: ConstVec<u8>, layout: &Layout) -> ConstVec<u8> {
const unsafe fn serialize_const_ptr(
ptr: *const (),
to: ConstVec<u8>,
layout: &Layout,
) -> ConstVec<u8> {
match layout {
Layout::Enum(layout) => serialize_const_enum(ptr, to, layout),
Layout::Struct(layout) => serialize_const_struct(ptr, to, layout),
Layout::Array(layout) => serialize_const_array(ptr, to, layout),
Layout::List(layout) => serialize_const_list(ptr, to, layout),
Layout::Primitive(layout) => serialize_const_primitive(ptr, to, layout),
}
@@ -710,156 +102,31 @@ const fn serialize_const_ptr(ptr: *const (), to: ConstVec<u8>, layout: &Layout)
/// b: 0x22,
/// c: 0x33333333,
/// }, buffer);
/// let buf = buffer.read();
/// assert_eq!(buf.as_ref(), &[0x11, 0x11, 0x11, 0x11, 0x22, 0x33, 0x33, 0x33, 0x33]);
/// assert_eq!(buffer.as_ref(), &[0xa3, 0x61, 0x61, 0x1a, 0x11, 0x11, 0x11, 0x11, 0x61, 0x62, 0x18, 0x22, 0x61, 0x63, 0x1a, 0x33, 0x33, 0x33, 0x33]);
/// ```
#[must_use = "The data is serialized into the returned buffer"]
pub const fn serialize_const<T: SerializeConst>(data: &T, to: ConstVec<u8>) -> ConstVec<u8> {
let ptr = data as *const T as *const ();
serialize_const_ptr(ptr, to, &T::MEMORY_LAYOUT)
}
/// Deserialize a primitive type into the out buffer at the offset passed in. Returns a new version of the buffer with the data added.
const fn deserialize_const_primitive<'a, const N: usize>(
mut from: ConstReadBuffer<'a>,
layout: &PrimitiveLayout,
out: (usize, [MaybeUninit<u8>; N]),
) -> Option<(ConstReadBuffer<'a>, [MaybeUninit<u8>; N])> {
let (start, mut out) = out;
let mut offset = 0;
while offset < layout.size {
// If the bytes are reversed, walk backwards from the end of the number when filling in bytes
let (from_new, value) = match from.get() {
Some(data) => data,
None => return None,
};
from = from_new;
if cfg!(any(target_endian = "big", feature = "test-big-endian")) {
out[start + layout.size - offset - 1] = MaybeUninit::new(value);
} else {
out[start + offset] = MaybeUninit::new(value);
}
offset += 1;
}
Some((from, out))
}
/// Deserialize a struct type into the out buffer at the offset passed in. Returns a new version of the buffer with the data added.
const fn deserialize_const_struct<'a, const N: usize>(
mut from: ConstReadBuffer<'a>,
layout: &StructLayout,
out: (usize, [MaybeUninit<u8>; N]),
) -> Option<(ConstReadBuffer<'a>, [MaybeUninit<u8>; N])> {
let (start, mut out) = out;
let mut i = 0;
while i < layout.data.len() {
// Deserialize the field at the offset pointer in the struct
let StructFieldLayout { offset, layout } = &layout.data[i];
let (new_from, new_out) = match deserialize_const_ptr(from, layout, (start + *offset, out))
{
Some(data) => data,
None => return None,
};
from = new_from;
out = new_out;
i += 1;
}
Some((from, out))
}
/// Deserialize an enum type into the out buffer at the offset passed in. Returns a new version of the buffer with the data added.
const fn deserialize_const_enum<'a, const N: usize>(
mut from: ConstReadBuffer<'a>,
layout: &EnumLayout,
out: (usize, [MaybeUninit<u8>; N]),
) -> Option<(ConstReadBuffer<'a>, [MaybeUninit<u8>; N])> {
let (start, mut out) = out;
let mut discriminant = 0;
// First, deserialize the discriminant
let mut offset = 0;
while offset < layout.discriminant.size {
// If the bytes are reversed, walk backwards from the end of the number when filling in bytes
let (from_new, value) = match from.get() {
Some(data) => data,
None => return None,
};
from = from_new;
if cfg!(target_endian = "big") {
out[start + layout.size - offset - 1] = MaybeUninit::new(value);
discriminant |= (value as u32) << ((layout.discriminant.size - offset - 1) * 8);
} else {
out[start + offset] = MaybeUninit::new(value);
discriminant |= (value as u32) << (offset * 8);
}
offset += 1;
}
// Then, deserialize the variant
let mut i = 0;
let mut matched_variant = false;
while i < layout.variants.len() {
// If the variant is the discriminated one, deserialize it
let EnumVariant { tag, data, .. } = &layout.variants[i];
if discriminant == *tag {
let offset = layout.variants_offset;
let (new_from, new_out) =
match deserialize_const_struct(from, data, (start + offset, out)) {
Some(data) => data,
None => return None,
};
from = new_from;
out = new_out;
matched_variant = true;
break;
}
i += 1;
}
if !matched_variant {
return None;
}
Some((from, out))
}
/// Deserialize a list type into the out buffer at the offset passed in. Returns a new version of the buffer with the data added.
const fn deserialize_const_list<'a, const N: usize>(
mut from: ConstReadBuffer<'a>,
layout: &ListLayout,
out: (usize, [MaybeUninit<u8>; N]),
) -> Option<(ConstReadBuffer<'a>, [MaybeUninit<u8>; N])> {
let (start, mut out) = out;
let len = layout.len;
let item_layout = layout.item_layout;
let mut i = 0;
while i < len {
let (new_from, new_out) =
match deserialize_const_ptr(from, item_layout, (start + i * item_layout.size(), out)) {
Some(data) => data,
None => return None,
};
from = new_from;
out = new_out;
i += 1;
}
Some((from, out))
// SAFETY: The pointer is valid and the layout is correct
unsafe { serialize_const_ptr(ptr, to, &T::MEMORY_LAYOUT) }
}
/// Deserialize a type into the out buffer at the offset passed in. Returns a new version of the buffer with the data added.
const fn deserialize_const_ptr<'a, const N: usize>(
from: ConstReadBuffer<'a>,
const fn deserialize_const_ptr<'a>(
from: &'a [u8],
layout: &Layout,
out: (usize, [MaybeUninit<u8>; N]),
) -> Option<(ConstReadBuffer<'a>, [MaybeUninit<u8>; N])> {
out: &mut [MaybeUninit<u8>],
) -> Option<&'a [u8]> {
match layout {
Layout::Enum(layout) => deserialize_const_enum(from, layout, out),
Layout::Struct(layout) => deserialize_const_struct(from, layout, out),
Layout::Array(layout) => deserialize_const_array(from, layout, out),
Layout::List(layout) => deserialize_const_list(from, layout, out),
Layout::Primitive(layout) => deserialize_const_primitive(from, layout, out),
}
}
/// Deserialize a type into the output buffer. Accepts `(type, ConstVec<u8>)` as input and returns `Option<(ConstReadBuffer, Instance of type)>`
/// Deserialize a type into the output buffer. Accepts `(type, ConstVec<u8>)` as input and returns `Option<(&'a [u8], Instance of type)>`
///
/// # Example
/// ```rust
@@ -879,7 +146,7 @@ const fn deserialize_const_ptr<'a, const N: usize>(
/// c: 0x33333333,
/// d: 0x44444444,
/// }, buffer);
/// let buf = buffer.read();
/// let buf = buffer.as_ref();
/// assert_eq!(deserialize_const!(Struct, buf).unwrap().1, Struct {
/// a: 0x11111111,
/// b: 0x22,
@@ -902,14 +169,13 @@ macro_rules! deserialize_const {
/// N must be `std::mem::size_of::<T>()`
#[must_use = "The data is deserialized from the input buffer"]
pub const unsafe fn deserialize_const_raw<const N: usize, T: SerializeConst>(
from: ConstReadBuffer,
) -> Option<(ConstReadBuffer, T)> {
from: &[u8],
) -> Option<(&[u8], T)> {
// Create uninitized memory with the size of the type
let out = [MaybeUninit::uninit(); N];
let mut out = [MaybeUninit::uninit(); N];
// Fill in the bytes into the buffer for the type
let (from, out) = match deserialize_const_ptr(from, &T::MEMORY_LAYOUT, (0, out)) {
Some(data) => data,
None => return None,
let Some(from) = deserialize_const_ptr(from, &T::MEMORY_LAYOUT, &mut out) else {
return None;
};
// Now that the memory is filled in, transmute it into the type
Some((from, unsafe {
packages/const-serialize/src/list.rs
+119
View File
@@ -0,0 +1,119 @@
use crate::*;
/// The layout for a dynamically sized list. The list layout is just a length and an item layout.
#[derive(Debug, Copy, Clone)]
pub struct ListLayout {
/// The size of the struct backing the list
pub(crate) size: usize,
/// The byte offset of the length field
len_offset: usize,
/// The layout of the length field
len_layout: PrimitiveLayout,
/// The byte offset of the data field
data_offset: usize,
/// The layout of the data field
data_layout: ArrayLayout,
}
impl ListLayout {
/// Create a new list layout
pub const fn new(
size: usize,
len_offset: usize,
len_layout: PrimitiveLayout,
data_offset: usize,
data_layout: ArrayLayout,
) -> Self {
Self {
size,
len_offset,
len_layout,
data_offset,
data_layout,
}
}
}
/// Serialize a dynamically sized list that is stored at the pointer passed in
pub(crate) const unsafe fn serialize_const_list(
ptr: *const (),
mut to: ConstVec<u8>,
layout: &ListLayout,
) -> ConstVec<u8> {
// Read the length of the list
let len_ptr = ptr.wrapping_byte_offset(layout.len_offset as _);
let len = layout.len_layout.read(len_ptr as *const u8) as usize;
let data_ptr = ptr.wrapping_byte_offset(layout.data_offset as _);
let item_layout = layout.data_layout.item_layout;
// If the item size is 1, deserialize as bytes directly
if item_layout.size() == 1 {
let slice = std::slice::from_raw_parts(data_ptr as *const u8, len);
to = write_bytes(to, slice);
}
// Otherwise, deserialize as a list of items
else {
let mut i = 0;
to = write_array(to, len);
while i < len {
let item = data_ptr.wrapping_byte_offset((i * item_layout.size()) as _);
to = serialize_const_ptr(item, to, item_layout);
i += 1;
}
}
to
}
/// Deserialize a list type into the out buffer at the offset passed in. Returns a new version of the buffer with the data added.
pub(crate) const fn deserialize_const_list<'a>(
from: &'a [u8],
layout: &ListLayout,
out: &mut [MaybeUninit<u8>],
) -> Option<&'a [u8]> {
let Some((_, len_out)) = out.split_at_mut_checked(layout.len_offset) else {
return None;
};
// If the list items are only one byte, serialize as bytes directly
let item_layout = layout.data_layout.item_layout;
if item_layout.size() == 1 {
let Ok((bytes, new_from)) = take_bytes(from) else {
return None;
};
// Write out the length of the list
layout.len_layout.write(bytes.len() as u32, len_out);
let Some((_, data_out)) = out.split_at_mut_checked(layout.data_offset) else {
return None;
};
let mut offset = 0;
while offset < bytes.len() {
data_out[offset] = MaybeUninit::new(bytes[offset]);
offset += 1;
}
Some(new_from)
}
// Otherwise, serialize as an list of objects
else {
let Ok((len, mut from)) = take_array(from) else {
return None;
};
// Write out the length of the list
layout.len_layout.write(len as u32, len_out);
let Some((_, mut data_out)) = out.split_at_mut_checked(layout.data_offset) else {
return None;
};
let mut i = 0;
while i < len {
let Some(new_from) = deserialize_const_ptr(from, item_layout, data_out) else {
return None;
};
let Some((_, item_out)) = data_out.split_at_mut_checked(item_layout.size()) else {
return None;
};
data_out = item_out;
from = new_from;
i += 1;
}
Some(from)
}
}
packages/const-serialize/src/primitive.rs
+121
View File
@@ -0,0 +1,121 @@
use crate::*;
use std::mem::MaybeUninit;
/// The layout for a primitive type. The bytes will be reversed if the target is big endian.
#[derive(Debug, Copy, Clone)]
pub struct PrimitiveLayout {
pub(crate) size: usize,
}
impl PrimitiveLayout {
/// Create a new primitive layout
pub const fn new(size: usize) -> Self {
Self { size }
}
/// Read the value from the given pointer
///
/// # Safety
/// The pointer must be valid for reads of `self.size` bytes.
pub const unsafe fn read(self, byte_ptr: *const u8) -> u32 {
let mut value = 0;
let mut offset = 0;
while offset < self.size {
// If the bytes are reversed, walk backwards from the end of the number when pushing bytes
let byte = if cfg!(target_endian = "big") {
unsafe {
byte_ptr
.wrapping_byte_add((self.size - offset - 1) as _)
.read()
}
} else {
unsafe { byte_ptr.wrapping_byte_add(offset as _).read() }
};
value |= (byte as u32) << (offset * 8);
offset += 1;
}
value
}
/// Write the value to the given buffer
pub const fn write(self, value: u32, out: &mut [MaybeUninit<u8>]) {
let bytes = value.to_ne_bytes();
let mut offset = 0;
while offset < self.size {
out[offset] = MaybeUninit::new(bytes[offset]);
offset += 1;
}
}
}
macro_rules! impl_serialize_const {
($type:ty) => {
unsafe impl SerializeConst for $type {
const MEMORY_LAYOUT: Layout = Layout::Primitive(PrimitiveLayout {
size: std::mem::size_of::<$type>(),
});
}
};
}
impl_serialize_const!(u8);
impl_serialize_const!(u16);
impl_serialize_const!(u32);
impl_serialize_const!(u64);
impl_serialize_const!(i8);
impl_serialize_const!(i16);
impl_serialize_const!(i32);
impl_serialize_const!(i64);
impl_serialize_const!(bool);
impl_serialize_const!(f32);
impl_serialize_const!(f64);
/// Serialize a primitive type that is stored at the pointer passed in
pub(crate) const unsafe fn serialize_const_primitive(
ptr: *const (),
to: ConstVec<u8>,
layout: &PrimitiveLayout,
) -> ConstVec<u8> {
let ptr = ptr as *const u8;
let mut offset = 0;
let mut i64_bytes = [0u8; 8];
while offset < layout.size {
// If the bytes are reversed, walk backwards from the end of the number when pushing bytes
let byte = unsafe {
if cfg!(any(target_endian = "big", feature = "test-big-endian")) {
ptr.wrapping_byte_offset((layout.size - offset - 1) as _)
.read()
} else {
ptr.wrapping_byte_offset(offset as _).read()
}
};
i64_bytes[offset] = byte;
offset += 1;
}
let number = i64::from_ne_bytes(i64_bytes);
write_number(to, number)
}
/// Deserialize a primitive type into the out buffer at the offset passed in. Returns a new version of the buffer with the data added.
pub(crate) const fn deserialize_const_primitive<'a>(
from: &'a [u8],
layout: &PrimitiveLayout,
out: &mut [MaybeUninit<u8>],
) -> Option<&'a [u8]> {
let mut offset = 0;
let Ok((number, from)) = take_number(from) else {
return None;
};
let bytes = number.to_le_bytes();
while offset < layout.size {
// If the bytes are reversed, walk backwards from the end of the number when filling in bytes
let byte = bytes[offset];
if cfg!(any(target_endian = "big", feature = "test-big-endian")) {
out[layout.size - offset - 1] = MaybeUninit::new(byte);
} else {
out[offset] = MaybeUninit::new(byte);
}
offset += 1;
}
Some(from)
}
packages/const-serialize/src/str.rs
+429
View File
@@ -0,0 +1,429 @@
use crate::*;
use std::{char, fmt::Debug, hash::Hash, mem::MaybeUninit};
const MAX_STR_SIZE: usize = 256;
/// A string that is stored in a constant sized buffer that can be serialized and deserialized at compile time
#[derive(Clone, Copy)]
pub struct ConstStr {
bytes: [MaybeUninit<u8>; MAX_STR_SIZE],
len: u32,
}
impl Debug for ConstStr {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ConstStr")
.field("str", &self.as_str())
.finish()
}
}
#[cfg(feature = "serde")]
mod serde_bytes {
use serde::{Deserialize, Serialize, Serializer};
use crate::ConstStr;
impl Serialize for ConstStr {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_str(self.as_str())
}
}
impl<'de> Deserialize<'de> for ConstStr {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
let s = String::deserialize(deserializer)?;
Ok(ConstStr::new(&s))
}
}
}
unsafe impl SerializeConst for ConstStr {
const MEMORY_LAYOUT: Layout = Layout::List(ListLayout::new(
std::mem::size_of::<Self>(),
std::mem::offset_of!(Self, len),
PrimitiveLayout {
size: std::mem::size_of::<u32>(),
},
std::mem::offset_of!(Self, bytes),
ArrayLayout {
len: MAX_STR_SIZE,
item_layout: &Layout::Primitive(PrimitiveLayout {
size: std::mem::size_of::<u8>(),
}),
},
));
}
#[cfg(feature = "const-serialize-07")]
unsafe impl const_serialize_07::SerializeConst for ConstStr {
const MEMORY_LAYOUT: const_serialize_07::Layout =
const_serialize_07::Layout::Struct(const_serialize_07::StructLayout::new(
std::mem::size_of::<Self>(),
&[
const_serialize_07::StructFieldLayout::new(
std::mem::offset_of!(Self, bytes),
const_serialize_07::Layout::List(const_serialize_07::ListLayout::new(
MAX_STR_SIZE,
&const_serialize_07::Layout::Primitive(
const_serialize_07::PrimitiveLayout::new(std::mem::size_of::<u8>()),
),
)),
),
const_serialize_07::StructFieldLayout::new(
std::mem::offset_of!(Self, len),
const_serialize_07::Layout::Primitive(
const_serialize_07::PrimitiveLayout::new(std::mem::size_of::<u32>()),
),
),
],
));
}
impl ConstStr {
/// Create a new constant string
pub const fn new(s: &str) -> Self {
let str_bytes = s.as_bytes();
// This is serialized as a constant sized array in const-serialize-07 which requires all memory to be initialized
let mut bytes = if cfg!(feature = "const-serialize-07") {
[MaybeUninit::new(0); MAX_STR_SIZE]
} else {
[MaybeUninit::uninit(); MAX_STR_SIZE]
};
let mut i = 0;
while i < str_bytes.len() {
bytes[i] = MaybeUninit::new(str_bytes[i]);
i += 1;
}
Self {
bytes,
len: str_bytes.len() as u32,
}
}
/// Get the bytes of the initialized portion of the string
const fn bytes(&self) -> &[u8] {
// Safety: All bytes up to the pointer are initialized
unsafe {
&*(self.bytes.split_at(self.len as usize).0 as *const [MaybeUninit<u8>]
as *const [u8])
}
}
/// Get a reference to the string
pub const fn as_str(&self) -> &str {
let str_bytes = self.bytes();
match std::str::from_utf8(str_bytes) {
Ok(s) => s,
Err(_) => panic!(
"Invalid utf8; ConstStr should only ever be constructed from valid utf8 strings"
),
}
}
/// Get the length of the string
pub const fn len(&self) -> usize {
self.len as usize
}
/// Check if the string is empty
pub const fn is_empty(&self) -> bool {
self.len == 0
}
/// Push a character onto the string
pub const fn push(self, byte: char) -> Self {
assert!(byte.is_ascii(), "Only ASCII bytes are supported");
let (bytes, len) = char_to_bytes(byte);
let (str, _) = bytes.split_at(len);
let Ok(str) = std::str::from_utf8(str) else {
panic!("Invalid utf8; char_to_bytes should always return valid utf8 bytes")
};
self.push_str(str)
}
/// Push a str onto the string
pub const fn push_str(self, str: &str) -> Self {
let Self { mut bytes, len } = self;
assert!(
str.len() + len as usize <= MAX_STR_SIZE,
"String is too long"
);
let str_bytes = str.as_bytes();
let new_len = len as usize + str_bytes.len();
let mut i = 0;
while i < str_bytes.len() {
bytes[len as usize + i] = MaybeUninit::new(str_bytes[i]);
i += 1;
}
Self {
bytes,
len: new_len as u32,
}
}
/// Split the string at a byte index. The byte index must be a char boundary
pub const fn split_at(self, index: usize) -> (Self, Self) {
let (left, right) = self.bytes().split_at(index);
let left = match std::str::from_utf8(left) {
Ok(s) => s,
Err(_) => {
panic!("Invalid utf8; you cannot split at a byte that is not a char boundary")
}
};
let right = match std::str::from_utf8(right) {
Ok(s) => s,
Err(_) => {
panic!("Invalid utf8; you cannot split at a byte that is not a char boundary")
}
};
(Self::new(left), Self::new(right))
}
/// Split the string at the last occurrence of a character
pub const fn rsplit_once(&self, char: char) -> Option<(Self, Self)> {
let str = self.as_str();
let mut index = str.len() - 1;
// First find the bytes we are searching for
let (char_bytes, len) = char_to_bytes(char);
let (char_bytes, _) = char_bytes.split_at(len);
let bytes = str.as_bytes();
// Then walk backwards from the end of the string
loop {
let byte = bytes[index];
// Look for char boundaries in the string and check if the bytes match
if let Some(char_boundary_len) = utf8_char_boundary_to_char_len(byte) {
// Split up the string into three sections: [before_char, in_char, after_char]
let (before_char, after_index) = bytes.split_at(index);
let (in_char, after_char) = after_index.split_at(char_boundary_len as usize);
if in_char.len() != char_boundary_len as usize {
panic!("in_char.len() should always be equal to char_boundary_len as usize")
}
// Check if the bytes for the current char and the target char match
let mut in_char_eq = true;
let mut i = 0;
let min_len = if in_char.len() < char_bytes.len() {
in_char.len()
} else {
char_bytes.len()
};
while i < min_len {
in_char_eq &= in_char[i] == char_bytes[i];
i += 1;
}
// If they do, convert the bytes to strings and return the split strings
if in_char_eq {
let Ok(before_char_str) = std::str::from_utf8(before_char) else {
panic!("Invalid utf8; utf8_char_boundary_to_char_len should only return Some when the byte is a character boundary")
};
let Ok(after_char_str) = std::str::from_utf8(after_char) else {
panic!("Invalid utf8; utf8_char_boundary_to_char_len should only return Some when the byte is a character boundary")
};
return Some((Self::new(before_char_str), Self::new(after_char_str)));
}
}
match index.checked_sub(1) {
Some(new_index) => index = new_index,
None => return None,
}
}
}
/// Split the string at the first occurrence of a character
pub const fn split_once(&self, char: char) -> Option<(Self, Self)> {
let str = self.as_str();
let mut index = 0;
// First find the bytes we are searching for
let (char_bytes, len) = char_to_bytes(char);
let (char_bytes, _) = char_bytes.split_at(len);
let bytes = str.as_bytes();
// Then walk forwards from the start of the string
while index < bytes.len() {
let byte = bytes[index];
// Look for char boundaries in the string and check if the bytes match
if let Some(char_boundary_len) = utf8_char_boundary_to_char_len(byte) {
// Split up the string into three sections: [before_char, in_char, after_char]
let (before_char, after_index) = bytes.split_at(index);
let (in_char, after_char) = after_index.split_at(char_boundary_len as usize);
if in_char.len() != char_boundary_len as usize {
panic!("in_char.len() should always be equal to char_boundary_len as usize")
}
// Check if the bytes for the current char and the target char match
let mut in_char_eq = true;
let mut i = 0;
let min_len = if in_char.len() < char_bytes.len() {
in_char.len()
} else {
char_bytes.len()
};
while i < min_len {
in_char_eq &= in_char[i] == char_bytes[i];
i += 1;
}
// If they do, convert the bytes to strings and return the split strings
if in_char_eq {
let Ok(before_char_str) = std::str::from_utf8(before_char) else {
panic!("Invalid utf8; utf8_char_boundary_to_char_len should only return Some when the byte is a character boundary")
};
let Ok(after_char_str) = std::str::from_utf8(after_char) else {
panic!("Invalid utf8; utf8_char_boundary_to_char_len should only return Some when the byte is a character boundary")
};
return Some((Self::new(before_char_str), Self::new(after_char_str)));
}
}
index += 1
}
None
}
}
impl PartialEq for ConstStr {
fn eq(&self, other: &Self) -> bool {
self.as_str() == other.as_str()
}
}
impl Eq for ConstStr {}
impl PartialOrd for ConstStr {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
impl Ord for ConstStr {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
self.as_str().cmp(other.as_str())
}
}
impl Hash for ConstStr {
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
self.as_str().hash(state);
}
}
#[test]
fn test_rsplit_once() {
let str = ConstStr::new("hello world");
assert_eq!(
str.rsplit_once(' '),
Some((ConstStr::new("hello"), ConstStr::new("world")))
);
let unicode_str = ConstStr::new("hi😀hello😀world😀world");
assert_eq!(
unicode_str.rsplit_once('😀'),
Some((ConstStr::new("hi😀hello😀world"), ConstStr::new("world")))
);
assert_eq!(unicode_str.rsplit_once('❌'), None);
for _ in 0..100 {
let random_str: String = (0..rand::random::<u8>() % 50)
.map(|_| rand::random::<char>())
.collect();
let konst = ConstStr::new(&random_str);
let mut seen_chars = std::collections::HashSet::new();
for char in random_str.chars().rev() {
let (char_bytes, len) = char_to_bytes(char);
let char_bytes = &char_bytes[..len];
assert_eq!(char_bytes, char.to_string().as_bytes());
if seen_chars.contains(&char) {
continue;
}
seen_chars.insert(char);
let (correct_left, correct_right) = random_str.rsplit_once(char).unwrap();
let (left, right) = konst.rsplit_once(char).unwrap();
println!("splitting {random_str:?} at {char:?}");
assert_eq!(left.as_str(), correct_left);
assert_eq!(right.as_str(), correct_right);
}
}
}
const CONTINUED_CHAR_MASK: u8 = 0b10000000;
const BYTE_CHAR_BOUNDARIES: [u8; 4] = [0b00000000, 0b11000000, 0b11100000, 0b11110000];
// Const version of https://doc.rust-lang.org/src/core/char/methods.rs.html#1765-1797
const fn char_to_bytes(char: char) -> ([u8; 4], usize) {
let code = char as u32;
let len = char.len_utf8();
let mut bytes = [0; 4];
match len {
1 => {
bytes[0] = code as u8;
}
2 => {
bytes[0] = ((code >> 6) & 0x1F) as u8 | BYTE_CHAR_BOUNDARIES[1];
bytes[1] = (code & 0x3F) as u8 | CONTINUED_CHAR_MASK;
}
3 => {
bytes[0] = ((code >> 12) & 0x0F) as u8 | BYTE_CHAR_BOUNDARIES[2];
bytes[1] = ((code >> 6) & 0x3F) as u8 | CONTINUED_CHAR_MASK;
bytes[2] = (code & 0x3F) as u8 | CONTINUED_CHAR_MASK;
}
4 => {
bytes[0] = ((code >> 18) & 0x07) as u8 | BYTE_CHAR_BOUNDARIES[3];
bytes[1] = ((code >> 12) & 0x3F) as u8 | CONTINUED_CHAR_MASK;
bytes[2] = ((code >> 6) & 0x3F) as u8 | CONTINUED_CHAR_MASK;
bytes[3] = (code & 0x3F) as u8 | CONTINUED_CHAR_MASK;
}
_ => panic!(
"encode_utf8: need more than 4 bytes to encode the unicode character, but the buffer has 4 bytes"
),
};
(bytes, len)
}
#[test]
fn fuzz_char_to_bytes() {
use std::char;
for _ in 0..100 {
let char = rand::random::<char>();
let (bytes, len) = char_to_bytes(char);
let str = std::str::from_utf8(&bytes[..len]).unwrap();
assert_eq!(char.to_string(), str);
}
}
const fn utf8_char_boundary_to_char_len(byte: u8) -> Option<u8> {
match byte {
0b00000000..=0b01111111 => Some(1),
0b11000000..=0b11011111 => Some(2),
0b11100000..=0b11101111 => Some(3),
0b11110000..=0b11111111 => Some(4),
_ => None,
}
}
#[test]
fn fuzz_utf8_byte_to_char_len() {
for _ in 0..100 {
let random_string: String = (0..rand::random::<u8>())
.map(|_| rand::random::<char>())
.collect();
let bytes = random_string.as_bytes();
let chars: std::collections::HashMap<_, _> = random_string.char_indices().collect();
for (i, byte) in bytes.iter().enumerate() {
match utf8_char_boundary_to_char_len(*byte) {
Some(char_len) => {
let char = chars
.get(&i)
.unwrap_or_else(|| panic!("{byte:b} is not a character boundary"));
assert_eq!(char.len_utf8(), char_len as usize);
}
None => {
assert!(!chars.contains_key(&i), "{byte:b} is a character boundary");
}
}
}
}
}
packages/const-serialize/src/struct.rs
+120
View File
@@ -0,0 +1,120 @@
use crate::*;
/// Plain old data for a field. Stores the offset of the field in the struct and the layout of the field.
#[derive(Debug, Copy, Clone)]
pub struct StructFieldLayout {
name: &'static str,
offset: usize,
layout: Layout,
}
impl StructFieldLayout {
/// Create a new struct field layout
pub const fn new(name: &'static str, offset: usize, layout: Layout) -> Self {
Self {
name,
offset,
layout,
}
}
}
/// Layout for a struct. The struct layout is just a list of fields with offsets
#[derive(Debug, Copy, Clone)]
pub struct StructLayout {
pub(crate) size: usize,
pub(crate) data: &'static [StructFieldLayout],
}
impl StructLayout {
/// Create a new struct layout
pub const fn new(size: usize, data: &'static [StructFieldLayout]) -> Self {
Self { size, data }
}
}
/// Serialize a struct that is stored at the pointer passed in
pub(crate) const unsafe fn serialize_const_struct(
ptr: *const (),
to: ConstVec<u8>,
layout: &StructLayout,
) -> ConstVec<u8> {
let mut i = 0;
let field_count = layout.data.len();
let mut to = write_map(to, field_count);
while i < field_count {
// Serialize the field at the offset pointer in the struct
let StructFieldLayout {
name,
offset,
layout,
} = &layout.data[i];
to = write_map_key(to, name);
let field = ptr.wrapping_byte_add(*offset as _);
to = serialize_const_ptr(field, to, layout);
i += 1;
}
to
}
/// Deserialize a struct type into the out buffer at the offset passed in. Returns a new version of the buffer with the data added.
pub(crate) const fn deserialize_const_struct<'a>(
from: &'a [u8],
layout: &StructLayout,
out: &mut [MaybeUninit<u8>],
) -> Option<&'a [u8]> {
let Ok((map, from)) = take_map(from) else {
return None;
};
let mut i = 0;
while i < layout.data.len() {
// Deserialize the field at the offset pointer in the struct
let StructFieldLayout {
name,
offset,
layout,
} = &layout.data[i];
let Ok(Some(from)) = map.find(name) else {
return None;
};
let Some((_, field_bytes)) = out.split_at_mut_checked(*offset) else {
return None;
};
if deserialize_const_ptr(from, layout, field_bytes).is_none() {
return None;
}
i += 1;
}
Some(from)
}
macro_rules! impl_serialize_const_tuple {
($($generic:ident: $generic_number:expr),*) => {
impl_serialize_const_tuple!(@impl ($($generic,)*) = $($generic: $generic_number),*);
};
(@impl $inner:ty = $($generic:ident: $generic_number:expr),*) => {
unsafe impl<$($generic: SerializeConst),*> SerializeConst for ($($generic,)*) {
const MEMORY_LAYOUT: Layout = {
Layout::Struct(StructLayout {
size: std::mem::size_of::<($($generic,)*)>(),
data: &[
$(
StructFieldLayout::new(stringify!($generic_number), std::mem::offset_of!($inner, $generic_number), $generic::MEMORY_LAYOUT),
)*
],
})
};
}
};
}
impl_serialize_const_tuple!(T1: 0);
impl_serialize_const_tuple!(T1: 0, T2: 1);
impl_serialize_const_tuple!(T1: 0, T2: 1, T3: 2);
impl_serialize_const_tuple!(T1: 0, T2: 1, T3: 2, T4: 3);
impl_serialize_const_tuple!(T1: 0, T2: 1, T3: 2, T4: 3, T5: 4);
impl_serialize_const_tuple!(T1: 0, T2: 1, T3: 2, T4: 3, T5: 4, T6: 5);
impl_serialize_const_tuple!(T1: 0, T2: 1, T3: 2, T4: 3, T5: 4, T6: 5, T7: 6);
impl_serialize_const_tuple!(T1: 0, T2: 1, T3: 2, T4: 3, T5: 4, T6: 5, T7: 6, T8: 7);
impl_serialize_const_tuple!(T1: 0, T2: 1, T3: 2, T4: 3, T5: 4, T6: 5, T7: 6, T8: 7, T9: 8);
impl_serialize_const_tuple!(T1: 0, T2: 1, T3: 2, T4: 3, T5: 4, T6: 5, T7: 6, T8: 7, T9: 8, T10: 9);