12 lines
435 B
Rust
12 lines
435 B
Rust
/// The character used to denote word breaks in the tokenized output.
|
|
pub(crate) const WORD_BREAK_CHAR: &str = "▁";
|
|
|
|
/// The token used to mark the start of a sentence.
|
|
pub(crate) const SENTENCE_START_TOKEN: &str = "<s>";
|
|
|
|
/// The token used to mark the end of a sentence.
|
|
pub(crate) const SENTENCE_END_TOKEN: &str = "</s>";
|
|
|
|
/// The token used to represent unknown words or subwords.
|
|
pub(crate) const UNKNOWN_TOKEN: &str = "<unk>";
|