chore: simplification, cleanup

This commit is contained in:
swaits committed 2024-09-25 21:10:26 -06:00
1 parent 608562c0c3
commit 485181549b
3 files changed
+25 -47

No files matched your search

+6 -2
View File
@@ -1,6 +1,6 @@
[package]
name = "bpetok"
version = "0.1.0"
version = "0.1.1"
edition = "2021"
authors = ["Stephen Waits <steve@waits.net>"]
description = "A simple CLI for tokenizing text input using Byte Pair Encoding (BPE)."
@@ -10,5 +10,9 @@ categories = ["command-line-utilities", "text-processing"]
[dependencies]
anyhow = "1.0.89"
bpe-tokenizer = { version = "0.1.4", features = ["default-small", "default-medium", "default-large"] }
bpe-tokenizer = { version = "0.1.4", features = [
"default-small",
"default-medium",
"default-large",
] }
clap = { version = "4.5.18", features = ["derive"] }