feat: add display UI with splash screen, dashboard, and RSSI sparkline
Implement full SSD1306 OLED display support: - Splash screen on boot (1.5s) with project name, version, board - Dashboard showing radio state, config, counters, RSSI/SNR - RSSI history sparkline (64-sample ring buffer, 2px-wide bars) - DisplayOn/DisplayOff host commands routed via dedicated channel - Pure render functions against generic DrawTarget (no hardware coupling) - Display on/off control bypasses radio task entirely Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
17624 files changed
+4564928
-38
No files matched your search
Vendored
+638
@@ -0,0 +1,638 @@
|
||||
//! Lex TOML tokens
|
||||
//!
|
||||
//! To get started, see [`Source::lex`][crate::Source::lex]
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(feature = "std")]
|
||||
mod test;
|
||||
mod token;
|
||||
|
||||
#[cfg(feature = "alloc")]
|
||||
use alloc::vec::Vec;
|
||||
|
||||
use winnow::stream::AsBStr as _;
|
||||
use winnow::stream::ContainsToken as _;
|
||||
use winnow::stream::FindSlice as _;
|
||||
use winnow::stream::Location;
|
||||
use winnow::stream::Stream as _;
|
||||
|
||||
use crate::Span;
|
||||
|
||||
pub use token::Token;
|
||||
pub use token::TokenKind;
|
||||
|
||||
/// Lex TOML [tokens][Token]
|
||||
///
|
||||
/// To get started, see [`Source::lex`][crate::Source::lex]
|
||||
pub struct Lexer<'i> {
|
||||
stream: Stream<'i>,
|
||||
eof: bool,
|
||||
}
|
||||
|
||||
impl<'i> Lexer<'i> {
|
||||
pub(crate) fn new(input: &'i str) -> Self {
|
||||
let mut stream = Stream::new(input);
|
||||
if input.as_bytes().starts_with(BOM) {
|
||||
let offset = BOM.len();
|
||||
#[cfg(feature = "unsafe")] // SAFETY: only called when next character is ASCII
|
||||
unsafe {
|
||||
stream.next_slice_unchecked(offset)
|
||||
};
|
||||
#[cfg(not(feature = "unsafe"))]
|
||||
stream.next_slice(offset);
|
||||
}
|
||||
Lexer { stream, eof: false }
|
||||
}
|
||||
|
||||
#[cfg(feature = "alloc")]
|
||||
pub fn into_vec(self) -> Vec<Token> {
|
||||
#![allow(unused_qualifications)] // due to MSRV of 1.66
|
||||
let capacity = core::cmp::min(
|
||||
self.stream.len(),
|
||||
usize::MAX / core::mem::size_of::<Token>(),
|
||||
);
|
||||
let mut vec = Vec::with_capacity(capacity);
|
||||
vec.extend(self);
|
||||
vec
|
||||
}
|
||||
}
|
||||
|
||||
impl Iterator for Lexer<'_> {
|
||||
type Item = Token;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
let Some(peek_byte) = self.stream.as_bstr().first() else {
|
||||
if self.eof {
|
||||
return None;
|
||||
} else {
|
||||
self.eof = true;
|
||||
let start = self.stream.current_token_start();
|
||||
let span = Span::new_unchecked(start, start);
|
||||
return Some(Token::new(TokenKind::Eof, span));
|
||||
}
|
||||
};
|
||||
Some(process_token(*peek_byte, &mut self.stream))
|
||||
}
|
||||
}
|
||||
|
||||
const BOM: &[u8] = b"\xEF\xBB\xBF";
|
||||
|
||||
pub(crate) type Stream<'i> = winnow::stream::LocatingSlice<&'i str>;
|
||||
|
||||
fn process_token(peek_byte: u8, stream: &mut Stream<'_>) -> Token {
|
||||
let token = match peek_byte {
|
||||
b'.' => lex_ascii_char(stream, TokenKind::Dot),
|
||||
b'=' => lex_ascii_char(stream, TokenKind::Equals),
|
||||
b',' => lex_ascii_char(stream, TokenKind::Comma),
|
||||
b'[' => lex_ascii_char(stream, TokenKind::LeftSquareBracket),
|
||||
b']' => lex_ascii_char(stream, TokenKind::RightSquareBracket),
|
||||
b'{' => lex_ascii_char(stream, TokenKind::LeftCurlyBracket),
|
||||
b'}' => lex_ascii_char(stream, TokenKind::RightCurlyBracket),
|
||||
b' ' => lex_whitespace(stream),
|
||||
b'\t' => lex_whitespace(stream),
|
||||
b'#' => lex_comment(stream),
|
||||
b'\r' => lex_crlf(stream),
|
||||
b'\n' => lex_ascii_char(stream, TokenKind::Newline),
|
||||
b'\'' => {
|
||||
if stream.starts_with(ML_LITERAL_STRING_DELIM) {
|
||||
lex_ml_literal_string(stream)
|
||||
} else {
|
||||
lex_literal_string(stream)
|
||||
}
|
||||
}
|
||||
b'"' => {
|
||||
if stream.starts_with(ML_BASIC_STRING_DELIM) {
|
||||
lex_ml_basic_string(stream)
|
||||
} else {
|
||||
lex_basic_string(stream)
|
||||
}
|
||||
}
|
||||
_ => lex_atom(stream),
|
||||
};
|
||||
token
|
||||
}
|
||||
|
||||
/// Process an ASCII character token
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// - `stream` must be UTF-8
|
||||
/// - `stream` must be non-empty
|
||||
/// - `stream[0]` must be ASCII
|
||||
fn lex_ascii_char(stream: &mut Stream<'_>, kind: TokenKind) -> Token {
|
||||
debug_assert!(!stream.is_empty());
|
||||
let start = stream.current_token_start();
|
||||
|
||||
let offset = 1; // an ascii character
|
||||
#[cfg(feature = "unsafe")] // SAFETY: only called when next character is ASCII
|
||||
unsafe {
|
||||
stream.next_slice_unchecked(offset)
|
||||
};
|
||||
#[cfg(not(feature = "unsafe"))]
|
||||
stream.next_slice(offset);
|
||||
|
||||
let end = stream.previous_token_end();
|
||||
let span = Span::new_unchecked(start, end);
|
||||
Token::new(kind, span)
|
||||
}
|
||||
|
||||
/// Process Whitespace
|
||||
///
|
||||
/// ```abnf
|
||||
/// ;; Whitespace
|
||||
///
|
||||
/// ws = *wschar
|
||||
/// wschar = %x20 ; Space
|
||||
/// wschar =/ %x09 ; Horizontal tab
|
||||
/// ```
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// - `stream` must be UTF-8
|
||||
/// - `stream` must be non-empty
|
||||
fn lex_whitespace(stream: &mut Stream<'_>) -> Token {
|
||||
debug_assert!(!stream.is_empty());
|
||||
let start = stream.current_token_start();
|
||||
|
||||
let offset = stream
|
||||
.as_bstr()
|
||||
.offset_for(|b| !WSCHAR.contains_token(b))
|
||||
.unwrap_or(stream.eof_offset());
|
||||
#[cfg(feature = "unsafe")] // SAFETY: WSCHAR ensures `offset` will be at UTF-8 boundary
|
||||
unsafe {
|
||||
stream.next_slice_unchecked(offset)
|
||||
};
|
||||
#[cfg(not(feature = "unsafe"))]
|
||||
stream.next_slice(offset);
|
||||
|
||||
let end = stream.previous_token_end();
|
||||
let span = Span::new_unchecked(start, end);
|
||||
Token::new(TokenKind::Whitespace, span)
|
||||
}
|
||||
|
||||
/// ```abnf
|
||||
/// wschar = %x20 ; Space
|
||||
/// wschar =/ %x09 ; Horizontal tab
|
||||
/// ```
|
||||
pub(crate) const WSCHAR: (u8, u8) = (b' ', b'\t');
|
||||
|
||||
/// Process Comment
|
||||
///
|
||||
/// ```abnf
|
||||
/// ;; Comment
|
||||
///
|
||||
/// comment-start-symbol = %x23 ; #
|
||||
/// non-ascii = %x80-D7FF / %xE000-10FFFF
|
||||
/// non-eol = %x09 / %x20-7E / non-ascii
|
||||
///
|
||||
/// comment = comment-start-symbol *non-eol
|
||||
/// ```
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// - `stream` must be UTF-8
|
||||
/// - `stream[0] == b'#'`
|
||||
fn lex_comment(stream: &mut Stream<'_>) -> Token {
|
||||
let start = stream.current_token_start();
|
||||
|
||||
let offset = stream
|
||||
.as_bytes()
|
||||
.find_slice((b'\r', b'\n'))
|
||||
.map(|s| s.start)
|
||||
.unwrap_or_else(|| stream.eof_offset());
|
||||
#[cfg(feature = "unsafe")] // SAFETY: newlines ensure `offset` is along UTF-8 boundary
|
||||
unsafe {
|
||||
stream.next_slice_unchecked(offset)
|
||||
};
|
||||
#[cfg(not(feature = "unsafe"))]
|
||||
stream.next_slice(offset);
|
||||
|
||||
let end = stream.previous_token_end();
|
||||
let span = Span::new_unchecked(start, end);
|
||||
Token::new(TokenKind::Comment, span)
|
||||
}
|
||||
|
||||
/// ```abnf
|
||||
/// comment-start-symbol = %x23 ; #
|
||||
/// ```
|
||||
pub(crate) const COMMENT_START_SYMBOL: u8 = b'#';
|
||||
|
||||
/// Process Newline
|
||||
///
|
||||
/// ```abnf
|
||||
/// ;; Newline
|
||||
///
|
||||
/// newline = %x0A ; LF
|
||||
/// newline =/ %x0D.0A ; CRLF
|
||||
/// ```
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// - `stream` must be UTF-8
|
||||
/// - `stream[0] == b'\r'`
|
||||
fn lex_crlf(stream: &mut Stream<'_>) -> Token {
|
||||
let start = stream.current_token_start();
|
||||
|
||||
let mut offset = '\r'.len_utf8();
|
||||
let has_lf = stream.as_bstr().get(1) == Some(&b'\n');
|
||||
if has_lf {
|
||||
offset += '\n'.len_utf8();
|
||||
}
|
||||
|
||||
#[cfg(feature = "unsafe")] // SAFETY: newlines ensure `offset` is along UTF-8 boundary
|
||||
unsafe {
|
||||
stream.next_slice_unchecked(offset)
|
||||
};
|
||||
#[cfg(not(feature = "unsafe"))]
|
||||
stream.next_slice(offset);
|
||||
let end = stream.previous_token_end();
|
||||
let span = Span::new_unchecked(start, end);
|
||||
|
||||
Token::new(TokenKind::Newline, span)
|
||||
}
|
||||
|
||||
/// Process literal string
|
||||
///
|
||||
/// ```abnf
|
||||
/// ;; Literal String
|
||||
///
|
||||
/// literal-string = apostrophe *literal-char apostrophe
|
||||
///
|
||||
/// apostrophe = %x27 ; ' apostrophe
|
||||
///
|
||||
/// literal-char = %x09 / %x20-26 / %x28-7E / non-ascii
|
||||
/// ```
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// - `stream` must be UTF-8
|
||||
/// - `stream[0] == b'\''`
|
||||
fn lex_literal_string(stream: &mut Stream<'_>) -> Token {
|
||||
let start = stream.current_token_start();
|
||||
|
||||
let offset = 1; // APOSTROPHE
|
||||
#[cfg(feature = "unsafe")] // SAFETY: only called when next character is ASCII
|
||||
unsafe {
|
||||
stream.next_slice_unchecked(offset)
|
||||
};
|
||||
#[cfg(not(feature = "unsafe"))]
|
||||
stream.next_slice(offset);
|
||||
|
||||
let offset = match stream.as_bstr().find_slice((APOSTROPHE, b'\n')) {
|
||||
Some(span) => {
|
||||
if stream.as_bstr()[span.start] == APOSTROPHE {
|
||||
span.end
|
||||
} else {
|
||||
span.start
|
||||
}
|
||||
}
|
||||
None => stream.eof_offset(),
|
||||
};
|
||||
#[cfg(feature = "unsafe")]
|
||||
// SAFETY: `APOSTROPHE`/newline ensure `offset` is along UTF-8 boundary
|
||||
unsafe {
|
||||
stream.next_slice_unchecked(offset)
|
||||
};
|
||||
#[cfg(not(feature = "unsafe"))]
|
||||
stream.next_slice(offset);
|
||||
|
||||
let end = stream.previous_token_end();
|
||||
let span = Span::new_unchecked(start, end);
|
||||
Token::new(TokenKind::LiteralString, span)
|
||||
}
|
||||
|
||||
/// ```abnf
|
||||
/// apostrophe = %x27 ; ' apostrophe
|
||||
/// ```
|
||||
pub(crate) const APOSTROPHE: u8 = b'\'';
|
||||
|
||||
/// Process multi-line literal string
|
||||
///
|
||||
/// ```abnf
|
||||
/// ;; Multiline Literal String
|
||||
///
|
||||
/// ml-literal-string = ml-literal-string-delim [ newline ] ml-literal-body
|
||||
/// ml-literal-string-delim
|
||||
/// ml-literal-string-delim = 3apostrophe
|
||||
/// ml-literal-body = *mll-content *( mll-quotes 1*mll-content ) [ mll-quotes ]
|
||||
///
|
||||
/// mll-content = literal-char / newline
|
||||
/// mll-quotes = 1*2apostrophe
|
||||
/// ```
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// - `stream` must be UTF-8
|
||||
/// - `stream.starts_with(ML_LITERAL_STRING_DELIM)`
|
||||
fn lex_ml_literal_string(stream: &mut Stream<'_>) -> Token {
|
||||
let start = stream.current_token_start();
|
||||
|
||||
let offset = ML_LITERAL_STRING_DELIM.len();
|
||||
#[cfg(feature = "unsafe")] // SAFETY: only called when next character is ASCII
|
||||
unsafe {
|
||||
stream.next_slice_unchecked(offset)
|
||||
};
|
||||
#[cfg(not(feature = "unsafe"))]
|
||||
stream.next_slice(offset);
|
||||
|
||||
let offset = match stream.as_bstr().find_slice(ML_LITERAL_STRING_DELIM) {
|
||||
Some(span) => span.end,
|
||||
None => stream.eof_offset(),
|
||||
};
|
||||
#[cfg(feature = "unsafe")]
|
||||
// SAFETY: `ML_LITERAL_STRING_DELIM` ensure `offset` is along UTF-8 boundary
|
||||
unsafe {
|
||||
stream.next_slice_unchecked(offset)
|
||||
};
|
||||
#[cfg(not(feature = "unsafe"))]
|
||||
stream.next_slice(offset);
|
||||
|
||||
if stream.as_bstr().peek_token() == Some(APOSTROPHE) {
|
||||
let offset = 1;
|
||||
#[cfg(feature = "unsafe")] // SAFETY: `APOSTROPHE` ensure `offset` is along UTF-8 boundary
|
||||
unsafe {
|
||||
stream.next_slice_unchecked(offset)
|
||||
};
|
||||
#[cfg(not(feature = "unsafe"))]
|
||||
stream.next_slice(offset);
|
||||
|
||||
if stream.as_bstr().peek_token() == Some(APOSTROPHE) {
|
||||
let offset = 1;
|
||||
#[cfg(feature = "unsafe")]
|
||||
// SAFETY: `APOSTROPHE` ensure `offset` is along UTF-8 boundary
|
||||
unsafe {
|
||||
stream.next_slice_unchecked(offset)
|
||||
};
|
||||
#[cfg(not(feature = "unsafe"))]
|
||||
stream.next_slice(offset);
|
||||
}
|
||||
}
|
||||
|
||||
let end = stream.previous_token_end();
|
||||
let span = Span::new_unchecked(start, end);
|
||||
Token::new(TokenKind::MlLiteralString, span)
|
||||
}
|
||||
|
||||
/// ```abnf
|
||||
/// ml-literal-string-delim = 3apostrophe
|
||||
/// ```
|
||||
pub(crate) const ML_LITERAL_STRING_DELIM: &str = "'''";
|
||||
|
||||
/// Process basic string
|
||||
///
|
||||
/// ```abnf
|
||||
/// ;; Basic String
|
||||
///
|
||||
/// basic-string = quotation-mark *basic-char quotation-mark
|
||||
///
|
||||
/// quotation-mark = %x22 ; "
|
||||
///
|
||||
/// basic-char = basic-unescaped / escaped
|
||||
/// basic-unescaped = wschar / %x21 / %x23-5B / %x5D-7E / non-ascii
|
||||
/// escaped = escape escape-seq-char
|
||||
///
|
||||
/// escape = %x5C ; \
|
||||
/// escape-seq-char = %x22 ; " quotation mark U+0022
|
||||
/// escape-seq-char =/ %x5C ; \ reverse solidus U+005C
|
||||
/// escape-seq-char =/ %x62 ; b backspace U+0008
|
||||
/// escape-seq-char =/ %x65 ; e escape U+001B
|
||||
/// escape-seq-char =/ %x66 ; f form feed U+000C
|
||||
/// escape-seq-char =/ %x6E ; n line feed U+000A
|
||||
/// escape-seq-char =/ %x72 ; r carriage return U+000D
|
||||
/// escape-seq-char =/ %x74 ; t tab U+0009
|
||||
/// escape-seq-char =/ %x78 2HEXDIG ; xHH U+00HH
|
||||
/// escape-seq-char =/ %x75 4HEXDIG ; uHHHH U+HHHH
|
||||
/// escape-seq-char =/ %x55 8HEXDIG ; UHHHHHHHH U+HHHHHHHH
|
||||
/// ```
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// - `stream` must be UTF-8
|
||||
/// - `stream[0] == b'"'`
|
||||
fn lex_basic_string(stream: &mut Stream<'_>) -> Token {
|
||||
let start = stream.current_token_start();
|
||||
|
||||
let offset = 1; // QUOTATION_MARK
|
||||
#[cfg(feature = "unsafe")] // SAFETY: only called when next character is ASCII
|
||||
unsafe {
|
||||
stream.next_slice_unchecked(offset)
|
||||
};
|
||||
#[cfg(not(feature = "unsafe"))]
|
||||
stream.next_slice(offset);
|
||||
|
||||
loop {
|
||||
// newline is present for error recovery
|
||||
match stream.as_bstr().find_slice((QUOTATION_MARK, ESCAPE, b'\n')) {
|
||||
Some(span) => {
|
||||
let found = stream.as_bstr()[span.start];
|
||||
if found == QUOTATION_MARK {
|
||||
let offset = span.end;
|
||||
#[cfg(feature = "unsafe")]
|
||||
// SAFETY: `QUOTATION_MARK` ensure `offset` is along UTF-8 boundary
|
||||
unsafe {
|
||||
stream.next_slice_unchecked(offset)
|
||||
};
|
||||
#[cfg(not(feature = "unsafe"))]
|
||||
stream.next_slice(offset);
|
||||
break;
|
||||
} else if found == ESCAPE {
|
||||
let offset = span.end;
|
||||
#[cfg(feature = "unsafe")]
|
||||
// SAFETY: `ESCAPE` / newline ensure `offset` is along UTF-8 boundary
|
||||
unsafe {
|
||||
stream.next_slice_unchecked(offset)
|
||||
};
|
||||
#[cfg(not(feature = "unsafe"))]
|
||||
stream.next_slice(offset);
|
||||
|
||||
let peek = stream.as_bstr().peek_token();
|
||||
match peek {
|
||||
Some(ESCAPE) | Some(QUOTATION_MARK) => {
|
||||
let offset = 1; // ESCAPE / QUOTATION_MARK
|
||||
#[cfg(feature = "unsafe")]
|
||||
#[cfg(feature = "unsafe")]
|
||||
// SAFETY: `ESCAPE` / newline ensure `offset` is along UTF-8 boundary
|
||||
unsafe {
|
||||
stream.next_slice_unchecked(offset)
|
||||
};
|
||||
#[cfg(not(feature = "unsafe"))]
|
||||
stream.next_slice(offset);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
continue;
|
||||
} else if found == b'\n' {
|
||||
let offset = span.start;
|
||||
#[cfg(feature = "unsafe")]
|
||||
// SAFETY: newline ensure `offset` is along UTF-8 boundary
|
||||
unsafe {
|
||||
stream.next_slice_unchecked(offset)
|
||||
};
|
||||
#[cfg(not(feature = "unsafe"))]
|
||||
stream.next_slice(offset);
|
||||
break;
|
||||
} else {
|
||||
unreachable!("found `{found}`");
|
||||
}
|
||||
}
|
||||
None => {
|
||||
stream.finish();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let end = stream.previous_token_end();
|
||||
let span = Span::new_unchecked(start, end);
|
||||
Token::new(TokenKind::BasicString, span)
|
||||
}
|
||||
|
||||
/// ```abnf
|
||||
/// quotation-mark = %x22 ; "
|
||||
/// ```
|
||||
pub(crate) const QUOTATION_MARK: u8 = b'"';
|
||||
|
||||
/// ```abnf
|
||||
/// escape = %x5C ; \
|
||||
/// ```
|
||||
pub(crate) const ESCAPE: u8 = b'\\';
|
||||
|
||||
/// Process multi-line basic string
|
||||
///
|
||||
/// ```abnf
|
||||
/// ;; Multiline Basic String
|
||||
///
|
||||
/// ml-basic-string = ml-basic-string-delim [ newline ] ml-basic-body
|
||||
/// ml-basic-string-delim
|
||||
/// ml-basic-string-delim = 3quotation-mark
|
||||
/// ml-basic-body = *mlb-content *( mlb-quotes 1*mlb-content ) [ mlb-quotes ]
|
||||
///
|
||||
/// mlb-content = basic-char / newline / mlb-escaped-nl
|
||||
/// mlb-quotes = 1*2quotation-mark
|
||||
/// mlb-escaped-nl = escape ws newline *( wschar / newline )
|
||||
/// ```
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// - `stream` must be UTF-8
|
||||
/// - `stream.starts_with(ML_BASIC_STRING_DELIM)`
|
||||
fn lex_ml_basic_string(stream: &mut Stream<'_>) -> Token {
|
||||
let start = stream.current_token_start();
|
||||
|
||||
let offset = ML_BASIC_STRING_DELIM.len();
|
||||
#[cfg(feature = "unsafe")] // SAFETY: only called when next character is ASCII
|
||||
unsafe {
|
||||
stream.next_slice_unchecked(offset)
|
||||
};
|
||||
#[cfg(not(feature = "unsafe"))]
|
||||
stream.next_slice(offset);
|
||||
|
||||
loop {
|
||||
// newline is present for error recovery
|
||||
match stream.as_bstr().find_slice((ML_BASIC_STRING_DELIM, "\\")) {
|
||||
Some(span) => {
|
||||
let found = stream.as_bstr()[span.start];
|
||||
if found == QUOTATION_MARK {
|
||||
let offset = span.end;
|
||||
#[cfg(feature = "unsafe")]
|
||||
// SAFETY: `QUOTATION_MARK` ensure `offset` is along UTF-8 boundary
|
||||
unsafe {
|
||||
stream.next_slice_unchecked(offset)
|
||||
};
|
||||
#[cfg(not(feature = "unsafe"))]
|
||||
stream.next_slice(offset);
|
||||
break;
|
||||
} else if found == ESCAPE {
|
||||
let offset = span.end;
|
||||
#[cfg(feature = "unsafe")]
|
||||
// SAFETY: `ESCAPE` ensure `offset` is along UTF-8 boundary
|
||||
unsafe {
|
||||
stream.next_slice_unchecked(offset)
|
||||
};
|
||||
#[cfg(not(feature = "unsafe"))]
|
||||
stream.next_slice(offset);
|
||||
|
||||
let peek = stream.as_bstr().peek_token();
|
||||
match peek {
|
||||
Some(ESCAPE) | Some(QUOTATION_MARK) => {
|
||||
let offset = 1; // ESCAPE / QUOTATION_MARK
|
||||
#[cfg(feature = "unsafe")]
|
||||
// SAFETY: `QUOTATION_MARK`/`ESCAPE` ensure `offset` is along UTF-8 boundary
|
||||
unsafe {
|
||||
stream.next_slice_unchecked(offset)
|
||||
};
|
||||
#[cfg(not(feature = "unsafe"))]
|
||||
stream.next_slice(offset);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
continue;
|
||||
} else {
|
||||
unreachable!("found `{found}`");
|
||||
}
|
||||
}
|
||||
None => {
|
||||
stream.finish();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if stream.as_bstr().peek_token() == Some(QUOTATION_MARK) {
|
||||
let offset = 1;
|
||||
#[cfg(feature = "unsafe")]
|
||||
// SAFETY: `QUOTATION_MARK` ensure `offset` is along UTF-8 boundary
|
||||
unsafe {
|
||||
stream.next_slice_unchecked(offset)
|
||||
};
|
||||
#[cfg(not(feature = "unsafe"))]
|
||||
stream.next_slice(offset);
|
||||
if stream.as_bstr().peek_token() == Some(QUOTATION_MARK) {
|
||||
let offset = 1;
|
||||
#[cfg(feature = "unsafe")]
|
||||
// SAFETY: `QUOTATION_MARK` ensure `offset` is along UTF-8 boundary
|
||||
unsafe {
|
||||
stream.next_slice_unchecked(offset)
|
||||
};
|
||||
#[cfg(not(feature = "unsafe"))]
|
||||
stream.next_slice(offset);
|
||||
}
|
||||
}
|
||||
|
||||
let end = stream.previous_token_end();
|
||||
let span = Span::new_unchecked(start, end);
|
||||
Token::new(TokenKind::MlBasicString, span)
|
||||
}
|
||||
|
||||
/// ```abnf
|
||||
/// ml-basic-string-delim = 3quotation-mark
|
||||
/// ```
|
||||
pub(crate) const ML_BASIC_STRING_DELIM: &str = "\"\"\"";
|
||||
|
||||
/// Process Atom
|
||||
///
|
||||
/// This is everything else
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// - `stream` must be UTF-8
|
||||
/// - `stream` must be non-empty
|
||||
fn lex_atom(stream: &mut Stream<'_>) -> Token {
|
||||
let start = stream.current_token_start();
|
||||
|
||||
// Intentionally leaves off quotes in case the opening quote was missing
|
||||
const TOKEN_START: &[u8] = b".=,[]{} \t#\r\n";
|
||||
let offset = stream
|
||||
.as_bstr()
|
||||
.offset_for(|b| TOKEN_START.contains_token(b))
|
||||
.unwrap_or_else(|| stream.eof_offset());
|
||||
#[cfg(feature = "unsafe")] // SAFETY: `TOKEN_START` ensure `offset` is along UTF-8 boundary
|
||||
unsafe {
|
||||
stream.next_slice_unchecked(offset)
|
||||
};
|
||||
#[cfg(not(feature = "unsafe"))]
|
||||
stream.next_slice(offset);
|
||||
|
||||
let end = stream.previous_token_end();
|
||||
let span = Span::new_unchecked(start, end);
|
||||
Token::new(TokenKind::Atom, span)
|
||||
}
|
||||
Vendored
+1700
File diff suppressed because it is too large.
Load diff
+106
@@ -0,0 +1,106 @@
|
||||
//! Lexed TOML tokens
|
||||
|
||||
use super::APOSTROPHE;
|
||||
use super::COMMENT_START_SYMBOL;
|
||||
use super::QUOTATION_MARK;
|
||||
use super::Span;
|
||||
use super::WSCHAR;
|
||||
use crate::decoder::Encoding;
|
||||
|
||||
/// An unvalidated TOML Token
|
||||
#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
|
||||
pub struct Token {
|
||||
pub(super) kind: TokenKind,
|
||||
pub(super) span: Span,
|
||||
}
|
||||
|
||||
impl Token {
|
||||
pub(super) fn new(kind: TokenKind, span: Span) -> Self {
|
||||
Self { kind, span }
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn kind(&self) -> TokenKind {
|
||||
self.kind
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn span(&self) -> Span {
|
||||
self.span
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
|
||||
#[repr(u8)]
|
||||
pub enum TokenKind {
|
||||
/// Either for dotted-key or float
|
||||
Dot = b'.',
|
||||
/// Key-value separator
|
||||
Equals = b'=',
|
||||
/// Value separator
|
||||
Comma = b',',
|
||||
/// Either array or standard-table start
|
||||
LeftSquareBracket = b'[',
|
||||
/// Either array or standard-table end
|
||||
RightSquareBracket = b']',
|
||||
/// Inline table start
|
||||
LeftCurlyBracket = b'{',
|
||||
/// Inline table end
|
||||
RightCurlyBracket = b'}',
|
||||
Whitespace = WSCHAR.0,
|
||||
Comment = COMMENT_START_SYMBOL,
|
||||
Newline = b'\n',
|
||||
LiteralString = APOSTROPHE,
|
||||
BasicString = QUOTATION_MARK,
|
||||
MlLiteralString = 1,
|
||||
MlBasicString,
|
||||
|
||||
/// Anything else
|
||||
Atom,
|
||||
|
||||
Eof,
|
||||
}
|
||||
|
||||
impl TokenKind {
|
||||
pub const fn description(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Dot => "`.`",
|
||||
Self::Equals => "`=`",
|
||||
Self::Comma => "`,`",
|
||||
Self::LeftSquareBracket => "`[`",
|
||||
Self::RightSquareBracket => "`]`",
|
||||
Self::LeftCurlyBracket => "`{`",
|
||||
Self::RightCurlyBracket => "`}`",
|
||||
Self::Whitespace => "whitespace",
|
||||
Self::Comment => "comment",
|
||||
Self::Newline => "newline",
|
||||
Self::LiteralString => "literal string",
|
||||
Self::BasicString => "basic string",
|
||||
Self::MlLiteralString => "multi-line literal string",
|
||||
Self::MlBasicString => "multi-line basic string",
|
||||
Self::Atom => "token",
|
||||
Self::Eof => "end-of-input",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn encoding(&self) -> Option<Encoding> {
|
||||
match self {
|
||||
Self::LiteralString => Some(Encoding::LiteralString),
|
||||
Self::BasicString => Some(Encoding::BasicString),
|
||||
Self::MlLiteralString => Some(Encoding::MlLiteralString),
|
||||
Self::MlBasicString => Some(Encoding::MlBasicString),
|
||||
Self::Atom
|
||||
| Self::LeftSquareBracket
|
||||
| Self::RightSquareBracket
|
||||
| Self::Dot
|
||||
| Self::Equals
|
||||
| Self::Comma
|
||||
| Self::RightCurlyBracket
|
||||
| Self::LeftCurlyBracket
|
||||
| Self::Whitespace
|
||||
| Self::Newline
|
||||
| Self::Comment
|
||||
| Self::Eof => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user