54 lines
1.5 KiB
TOML
54 lines
1.5 KiB
TOML
[package]
|
|
name = "bpe-tokenizer"
|
|
version = "0.1.3"
|
|
authors = ["Stephen Waits <steve@waits.net>"]
|
|
edition = "2021"
|
|
description = "A BPE Tokenizer library."
|
|
keywords = ["byte", "pair", "encoding", "bpe", "tokenizer"]
|
|
categories = ["algorithms", "text-processing"]
|
|
documentation = "https://docs.rs/bpe-tokenizer/"
|
|
readme = "README.md"
|
|
homepage = "https://github.com/swaits/bpe-tokenizer/"
|
|
repository = "https://github.com/swaits/bpe-tokenizer/"
|
|
rust-version = "1.64"
|
|
license-file = "LICENSE"
|
|
exclude = ["vocab/multi.wiki.bpe.vs1000000.vocab"]
|
|
# ^ default-large vocab is temporarily excluded until we can get a crates.io exception
|
|
|
|
[dependencies]
|
|
thiserror = "1.0.64"
|
|
unicode-segmentation = "1.12.0"
|
|
|
|
# optional, only needed when defaul-{small,medium,large} are enabled
|
|
bincode = { version = "1.3.3", optional = true }
|
|
lz4_flex = { version = "0.11.3", optional = true }
|
|
serde = { version = "1.0.210", optional = true }
|
|
|
|
[features]
|
|
default-small = ["bincode", "lz4_flex", "serde"]
|
|
default-medium = ["bincode", "lz4_flex", "serde"]
|
|
default-large = ["bincode", "lz4_flex", "serde"]
|
|
|
|
[build-dependencies]
|
|
bincode = { version = "1.3.3", optional = true }
|
|
lz4_flex = { version = "0.11.3", optional = true }
|
|
serde = { version = "1.0.210", optional = true }
|
|
|
|
# **DISABLED** until we can get a crates.io exception
|
|
# [package.metadata.docs.rs]
|
|
# all-features = true
|
|
|
|
[profile.dev]
|
|
opt-level = 1
|
|
debug = true
|
|
overflow-checks = true
|
|
|
|
[profile.release]
|
|
opt-level = 3
|
|
lto = "thin"
|
|
codegen-units = 1
|
|
debug = false
|
|
panic = "abort"
|
|
overflow-checks = false
|
|
strip = "symbols"
|