diff --git a/Cargo.toml b/Cargo.toml index de0bd6d..8716ae2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,8 @@ homepage = "https://github.com/swaits/bpe-tokenizer/" repository = "https://github.com/swaits/bpe-tokenizer/" rust-version = "1.64" license-file = "LICENSE" +exclude = ["vocab/multi.wiki.bpe.vs1000000.vocab"] +# ^ default-large vocab is temporarily excluded until we can get a crates.io exception [dependencies] thiserror = "1.0.64" diff --git a/README.md b/README.md index 3c20c29..aa92c41 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,9 @@ frequent token pairs. It supports **Unicode-aware** text segmentation for sentence and word splitting, making it suitable for processing a variety of languages and scripts. +**NOTE:** The `default-large` feature is temporarily disabled due to crate size +limitations; presently awaiting an exception from . + ## Features - **Bring your own BPE token vocabularies**, or use ...