20 lines
592 B
TOML
20 lines
592 B
TOML
[package]
|
|
name = "bpetok"
|
|
version = "0.1.2"
|
|
edition = "2021"
|
|
authors = ["Stephen Waits <steve@waits.net>"]
|
|
description = "A simple CLI for tokenizing text input using Byte Pair Encoding (BPE)."
|
|
license-file = "LICENSE"
|
|
keywords = ["text", "tokenizer", "bpe", "tokenization", "cli"]
|
|
categories = ["command-line-utilities", "text-processing"]
|
|
repository = "https://git.swaits.com/swaits/bpetok"
|
|
|
|
[dependencies]
|
|
anyhow = "1.0.89"
|
|
bpe-tokenizer = { version = "0.1.4", features = [
|
|
"default-small",
|
|
"default-medium",
|
|
"default-large",
|
|
] }
|
|
clap = { version = "4.5.18", features = ["derive"] }
|