15 lines
531 B
TOML
15 lines
531 B
TOML
[package]
|
|
name = "bpetok"
|
|
version = "0.1.0"
|
|
edition = "2021"
|
|
authors = ["Stephen Waits <steve@waits.net>"]
|
|
description = "A simple CLI for tokenizing text input using Byte Pair Encoding (BPE)."
|
|
license-file = "LICENSE"
|
|
keywords = ["text", "tokenizer", "bpe", "tokenization", "cli"]
|
|
categories = ["command-line-utilities", "text-processing"]
|
|
|
|
[dependencies]
|
|
anyhow = "1.0.89"
|
|
bpe-tokenizer = { version = "0.1.4", features = ["default-small", "default-medium", "default-large"] }
|
|
clap = { version = "4.5.18", features = ["derive"] }
|