swaits/wanderbooks
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

P2 slice 5: files in and out

Browse Source 
Files reach the books through the server: uploads by hash and size,
then filed; exports (the CPA package, the 1099 worksheet, W-9s) handed
over as downloads from the server, never read off disk by the agent.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01T3gXMVt7XJPi6ens1or9Dj
This commit is contained in:
swaitsandClaude Opus 5.5 committed 2026-10-20 11:14:21 -06:00
1 parent d1f3ad891a
commit b26c099a68
44 files changed
+5075 -2739
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

crates/ledger-mcp/src/lib.rs
+11 -50
View File
@@ -3,8 +3,7 @@
//! [`run`] is the whole server; the `ledger-mcp` binary and `wanderbooks
//! dev-stdio` (one executable, plan D-22) both call it.
//!
//! Usage: `ledger-mcp [--data-dir PATH] [--config PATH] [--books-toml PATH]
//! [--import-root DIR]...`
//! Usage: `ledger-mcp [--data-dir PATH] [--config PATH] [--books-toml PATH]`
//!
//! The data dir holds `journal.jsonl` (the source of truth), `wanderbooks.db`
//! (the disposable SQLite projection), `blobs/` (content-addressed receipts),
@@ -12,19 +11,16 @@
//! configured base currency. Company settings (name, fiscal year start,
//! receipt threshold) come from the books dir's `wanderbooks.toml`, by
//! default the one next to the data dir, and reload whenever it changes
//! (settings.rs). `books.import_local_file` and `books.set_w9`'s
//! `w9_file_path` read only from the import roots: each `--import-root`,
//! or by default the downloads folder (local_files.rs); a root that is
//! `/`, the home folder, or holds the books, the vault or a key is refused
//! at startup.
//! (settings.rs). Files come in only as tool input (inline, at most 256
//! KiB): uploads need the books server's socket (`wanderbooks mcp`).
pub mod access;
pub mod admin;
mod handbook;
#[cfg(unix)]
pub mod local;
mod local_files;
pub mod opening;
mod paths;
mod policy;
pub mod scopes;
mod server;
@@ -35,7 +31,6 @@ mod wire;
use ledger_core::Currency;
use ledger_store::Store;
use local_files::ImportRoots;
use policy::ReceiptPolicy;
use rmcp::ServiceExt;
use serde::Deserialize;
@@ -59,7 +54,7 @@ fn default_data_dir() -> PathBuf {
fn print_usage(prog: &str) {
eprintln!(
"usage: {prog} [--data-dir PATH] [--config PATH] [--books-toml PATH] [--vault-key-file PATH] [--http ADDR] [--import-root DIR]..."
"usage: {prog} [--data-dir PATH] [--config PATH] [--books-toml PATH] [--vault-key-file PATH] [--http ADDR]"
);
eprintln!(" --data-dir directory holding journal.jsonl (default ~/.local/share/wanderbooks)");
eprintln!(" --config JSON config file (default <data-dir>/config.json, optional)");
@@ -72,13 +67,10 @@ fn print_usage(prog: &str) {
eprintln!(
" --http serve over Streamable HTTP at ADDR (e.g. 127.0.0.1:8080) instead of stdio"
);
eprintln!(
" --import-root a folder books.import_local_file may read from (repeatable; default: the downloads folder)"
);
}
/// How [`run`] stops when it can't go on: the reason is already printed
/// on stderr, and the number is the process's exit code (2: a bad argument or a refused import folder; 1: books from a newer wanderbooks). A
/// on stderr, and the number is the process's exit code (2: a bad argument; 1: books from a newer wanderbooks). A
/// library never ends the process itself, so the books it opened are
/// closed properly on the way out; the binaries turn this into the exit.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -95,8 +87,8 @@ impl std::error::Error for Exit {}
/// Run the books server with `args` (the command line after the program
/// name). `prog` names the program in the usage text: `ledger-mcp`, or
/// `wanderbooks dev-stdio`. It returns when the client disconnects; a bad
/// argument, books from a newer wanderbooks, or a refused import root
/// return [`Exit`] after a plain message.
/// argument or books from a newer wanderbooks return [`Exit`] after a
/// plain message.
pub fn run(
prog: &str,
args: impl IntoIterator<Item = String>,
@@ -106,7 +98,6 @@ pub fn run(
let mut books_toml: Option<PathBuf> = None;
let mut vault_key_file: Option<PathBuf> = None;
let mut http_addr: Option<String> = None;
let mut import_roots: Vec<PathBuf> = Vec::new();
let mut args = args.into_iter();
while let Some(arg) = args.next() {
match arg.as_str() {
@@ -129,11 +120,6 @@ pub fn run(
"--http" => {
http_addr = Some(args.next().ok_or("--http needs an address")?);
}
"--import-root" => {
import_roots.push(PathBuf::from(
args.next().ok_or("--import-root needs a folder")?,
));
}
"--help" | "-h" => {
print_usage(prog);
return Ok(());
@@ -153,11 +139,6 @@ pub fn run(
);
return Err(Box::new(Exit(2)));
}
let import_roots = if import_roots.is_empty() {
ImportRoots::defaults()
} else {
ImportRoots::new(import_roots)
};
let config_path = config_path.unwrap_or_else(|| data_dir.join("config.json"));
// config.json's base currency matters only to books being created: it
// is read then, and never otherwise (books that keep their own
@@ -232,36 +213,18 @@ pub fn run(
}
);
let live = std::sync::Arc::new(std::sync::Mutex::new(live));
// No import root may be `/`, the home folder, or hold the books, the
// vault or a key (local_files.rs INVARIANT 0): a named one stops the
// server here, a default one is dropped.
let settings_now = live.lock().unwrap_or_else(|e| e.into_inner()).current();
let import_roots = match server::guard_import_roots(
&store,
&settings_now,
&server::Layout::BooksDir,
import_roots,
) {
Ok(roots) => roots,
Err(why) => {
eprintln!("ledger-mcp: {why}");
return Err(Box::new(Exit(2)));
}
};
let rt = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()?;
if let Some(addr) = http_addr {
eprintln!("ledger-mcp: store verified, serving Streamable HTTP on {addr}");
rt.block_on(serve_http(addr, store, live, import_roots))?;
rt.block_on(serve_http(addr, store, live))?;
return Ok(());
}
eprintln!("ledger-mcp: store verified, serving over stdio");
let server = BooksServer::new(store, ReceiptPolicy::default())
.with_live_settings(live)
.with_import_roots(import_roots);
let server = BooksServer::new(store, ReceiptPolicy::default()).with_live_settings(live);
rt.block_on(async {
// The wire answers what JSON-RPC requires that rmcp drops (wire.rs).
let (input, output, writer) = wire::spawn(tokio::io::stdin(), tokio::io::stdout());
@@ -277,7 +240,6 @@ async fn serve_http(
addr: String,
store: Store,
live: std::sync::Arc<std::sync::Mutex<settings::Live>>,
import_roots: ImportRoots,
) -> Result<(), Box<dyn std::error::Error>> {
use rmcp::transport::streamable_http_server::{
StreamableHttpService, session::local::LocalSessionManager,
@@ -290,8 +252,7 @@ async fn serve_http(
let service = StreamableHttpService::new(
move || {
let server = server::BooksServer::new_shared(store.clone(), ReceiptPolicy::default())
.with_live_settings(live.clone())
.with_import_roots(import_roots.clone());
.with_live_settings(live.clone());
Ok(server)
},
Arc::new(LocalSessionManager::default()),
crates/ledger-mcp/src/local.rs
+450 -67
View File
@@ -43,10 +43,20 @@
//! bindings a new session is refused (an initialize), never a binding
//! dropped. Tokens are made, listed and revoked only through
//! the admin socket.
//! 9. A file's bytes come in only by `PUT /b/{id}/u/{upload}` and go
//! out only by `GET /b/{id}/a/{attachment}` (plan 4.7), each through
//! the same door as 8 and counted for the close like any request:
//! an upload only from the token that began it, a download only with
//! books:read. Each, refused at the door or not, is recorded in the
//! book's access log as `upload` or `download` with its id, and the
//! bytes (kept, or read) and their record are one task that runs to
//! its end if the client hangs up (a process killed in that task
//! excepted); bytes whose record can't be written aren't kept, and
//! the books stop. Lapsed uploads are swept
//! every SWEEP_EVERY. Nothing comes in once the books stopped.
//! ```
use crate::admin::{self, Request, Response};
use crate::local_files::ImportRoots;
use crate::policy::ReceiptPolicy;
use crate::server::{BooksServer, Layout, SharedStore};
use crate::settings::Live;
@@ -75,10 +85,9 @@ pub const SERVER_LOCK: &str = "server.lock";
/// What the server prints on stdout when it is ready (INVARIANT 5).
pub const READY: &str = "ready";
/// How large a request may be. Receipts still ride inside tool calls as
/// base64 until uploads have their own route (P2 slice 5, which brings
/// this down to 1 MiB).
pub const MAX_BODY: usize = 64 * 1024 * 1024;
/// How large an MCP request may be (plan 3.2): files of any size come
/// through uploads (`/b/{book}/u/{upload}`); inline, at most 256 KiB.
pub const MAX_BODY: usize = 1024 * 1024;
/// How long the admin socket waits for a request to arrive.
const ADMIN_READ_TIMEOUT: Duration = Duration::from_secs(30);
@@ -89,6 +98,9 @@ use ledger_store::root::THE_ROOT;
/// didn't open, books that don't exist): a client knows it didn't run.
pub const REFUSED_HEADER: &str = "x-wanderbooks-refused";
/// How often each book's lapsed uploads are swept.
const SWEEP_EVERY: Duration = Duration::from_secs(10);
type Service = StreamableHttpService<BooksServer, LocalSessionManager>;
/// A book the server holds.
@@ -103,6 +115,8 @@ enum Book {
/// Its access log, where requests refused at the door are
/// recorded too (access.rs).
access: Arc<AccessLog>,
/// Its files on their way in (`/b/{id}/u/{upload}`).
uploads: ledger_store::uploads::Uploads,
},
/// Not opened, with why (INVARIANT 3).
Failed(String),
@@ -121,7 +135,6 @@ impl Book {
/// Every book the server holds, and what serving them needs.
struct Books {
root: Root,
import_roots: ImportRoots,
held: RwLock<BTreeMap<String, Book>>,
/// Ends every MCP session (INVARIANT 6, last).
sessions: CancellationToken,
@@ -250,17 +263,14 @@ impl Books {
let mut live = Live::own();
live.sync(store.ledger().settings(), store.base_currency().code());
let layout = Layout::Root(self.root.path().to_path_buf());
let roots = crate::server::guard_import_roots(
&store,
&live.current(),
&layout,
self.import_roots.clone(),
)
.map_err(|e| self.hide(e))?;
let access = Arc::new(
AccessLog::open(store.data_dir())
.map_err(|e| format!("its access log can't be used: {e}"))?,
);
let uploads =
ledger_store::uploads::Uploads::open(store.data_dir(), chrono::Utc::now().timestamp())
.map_err(|e| self.hide(e.to_string()))?;
let server_uploads = uploads.clone();
let live = Arc::new(Mutex::new(live));
let store = SharedStore::new(store);
let log = access.clone();
@@ -276,9 +286,9 @@ impl Books {
Ok(
BooksServer::new_shared(store.clone(), ReceiptPolicy::default())
.with_live_settings(live.clone())
.with_import_roots(roots.clone())
.with_layout(layout.clone())
.with_access(log.clone()),
.with_access(log.clone())
.with_uploads(server_uploads.clone()),
)
},
Arc::new(sessions()),
@@ -288,6 +298,7 @@ impl Books {
service,
store: shared,
access,
uploads,
})
}
@@ -848,12 +859,29 @@ const SESSION_HEADER: &str = "mcp-session-id";
/// live sessions).
const MAX_SESSIONS: usize = 100_000;
/// What a request asks for, as the access log names it: an MCP request
/// (action `connect`, its method), an upload (`upload`, its id) or a
/// download (`download`, the attachment's id).
struct Asked {
action: &'static str,
subject: String,
}
impl Asked {
fn connect(method: &axum::http::Method) -> Self {
Self {
action: "connect",
subject: method.as_str().to_string(),
}
}
}
/// A request refused at the door: recorded in the book's access log
/// (access.rs INVARIANT 3), then answered with `status` and `why`.
async fn refused_at_door(
book: &Book,
principal: Principal,
method: &axum::http::Method,
asked: &Asked,
code: &'static str,
status: axum::http::StatusCode,
why: String,
@@ -865,8 +893,9 @@ async fn refused_at_door(
principal,
client: None,
request_id: None,
action: "connect".into(),
subject: method.as_str().to_string(),
action: asked.action.into(),
// Masked, then cut, by the log itself (access.rs INVARIANT 2).
subject: asked.subject.clone(),
args_digest: None,
result: Outcome::Refused(code.into()),
head_after: None,
@@ -892,34 +921,31 @@ async fn refused_at_door(
response
}
/// `/b/{book}/mcp`: the book's MCP service, for a request carrying a token
/// for that book (INVARIANT 8).
async fn mcp(
axum::extract::State(books): axum::extract::State<Arc<Books>>,
axum::extract::Path(book): axum::extract::Path<String>,
mut request: axum::extract::Request,
) -> axum::response::Response {
/// The door to `book` (INVARIANT 8): the books held open, the server not
/// closing, and a live token for those books; else the answer refusing
/// the request (recorded in the books' access log, where there is one).
async fn door(
books: &Arc<Books>,
book: &str,
asked: &Asked,
headers: &axum::http::HeaderMap,
) -> Result<(Book, Token), Box<axum::response::Response>> {
use axum::http::StatusCode;
use axum::response::IntoResponse;
use tower::ServiceExt;
// Counted before the closing check, so closing never misses it.
let counted = (request.method() != axum::http::Method::GET).then(|| InFlight::new(&books));
books
.last_arrival
.store(books.started.elapsed().as_millis() as u64, Ordering::SeqCst);
let closing = books.closing.is_cancelled();
let found = books.held().get(&book).cloned();
let (service, held) = match found {
let found = books.held().get(book).cloned();
let held = match found {
Some(Book::Failed(why)) if !closing => {
return (
return Err((
StatusCode::SERVICE_UNAVAILABLE,
[(REFUSED_HEADER, "not-open")],
format!("the books {book} didn't open: {why}"),
)
.into_response();
.into_response()
.into());
}
None if !closing => {
return (
return Err((
StatusCode::NOT_FOUND,
[(REFUSED_HEADER, "no-such-books")],
format!(
@@ -927,22 +953,20 @@ async fn mcp(
book.chars().take(64).collect::<String>()
),
)
.into_response();
.into_response()
.into());
}
Some(b @ Book::Open { .. }) => match &b {
Book::Open { service, .. } => (service.clone(), b.clone()),
Book::Failed(_) => unreachable!(),
},
Some(b @ Book::Open { .. }) => b,
_ => {
return (
return Err((
StatusCode::SERVICE_UNAVAILABLE,
"the books are closing: this request wasn't run",
)
.into_response();
.into_response()
.into());
}
};
let method = request.method().clone();
let token = books.bearer(request.headers());
let token = books.bearer(headers);
let principal = |t: &Token| Principal {
id: t.id.clone(),
kind: "agent".into(),
@@ -952,41 +976,382 @@ async fn mcp(
let who = token
.as_ref()
.map_or_else(|_| Principal::unknown(), principal);
return refused_at_door(
return Err(refused_at_door(
&held,
who,
&method,
asked,
crate::access::CLOSING,
StatusCode::SERVICE_UNAVAILABLE,
"the books are closing: this request wasn't run".into(),
)
.await;
.await
.into());
}
let token = match token {
Ok(t) => t,
Err((code, why)) => {
return refused_at_door(
return Err(refused_at_door(
&held,
Principal::unknown(),
&method,
asked,
code,
StatusCode::UNAUTHORIZED,
why,
)
.await;
.await
.into());
}
};
if token.book != book {
return refused_at_door(
return Err(refused_at_door(
&held,
principal(&token),
&method,
asked,
"wrong_books",
StatusCode::FORBIDDEN,
format!("this token is for the books {}, not {book}", token.book),
)
.await;
.await
.into());
}
Ok((held, token))
}
/// `PUT /b/{book}/u/{upload}`: the bytes of an upload begun with
/// `books.begin_upload`, from the token that began it (INVARIANT 9; plan
/// 4.7): at most the upload's limit, kept only if exactly the size and
/// SHA-256 declared, once; recorded in the access log (action `upload`).
async fn upload(
axum::extract::State(books): axum::extract::State<Arc<Books>>,
axum::extract::Path((book, id)): axum::extract::Path<(String, String)>,
request: axum::extract::Request,
) -> axum::response::Response {
use axum::http::StatusCode;
use axum::response::IntoResponse;
use ledger_store::uploads::UploadError;
let _counted = InFlight::new(&books);
books
.last_arrival
.store(books.started.elapsed().as_millis() as u64, Ordering::SeqCst);
let asked = Asked {
action: "upload",
subject: id.clone(),
};
let (held, token) = match door(&books, &book, &asked, request.headers()).await {
Ok(passed) => passed,
Err(refused) => {
// What was sent is read and dropped (up to the most an upload
// can be: only this account reaches the socket), so the client
// gets the refusal, not a connection cut mid-send.
use futures_util::StreamExt;
let mut rest = request.into_body().into_data_stream();
let mut read = 0u64;
while let Some(Ok(chunk)) = rest.next().await {
read += chunk.len() as u64;
if read > ledger_store::uploads::MAX_UPLOAD {
break;
}
}
return *refused;
}
};
let Book::Open {
access,
uploads,
store,
..
} = &held
else {
unreachable!("the door lets only open books through")
};
// Streamed to the disk as it comes, never held whole; `receive` reads
// no more than the size declared and one byte. The bytes and their
// record are one blocking task, which runs to its end even if the
// client hangs up: never bytes kept without their record, and a close
// waits for both (the count goes with them).
use futures_util::TryStreamExt;
let stream = request
.into_body()
.into_data_stream()
.map_err(std::io::Error::other);
let mut body = tokio_util::io::SyncIoBridge::new(tokio_util::io::StreamReader::new(stream));
let (uploads, access, stopped) = (uploads.clone(), access.clone(), store.is_stopped());
let task = tokio::task::spawn_blocking(move || {
let _counted = _counted;
// The books server stopped after an internal error: nothing more
// comes in (SharedStore INVARIANT 2).
let received = if stopped {
Err(UploadError::Bad(crate::server::STOPPED.into()))
} else {
uploads.receive(&id, &token.id, &mut body, chrono::Utc::now().timestamp())
};
// Refused: the rest of what was sent (up to the most an upload
// can be) is read and dropped, so the client gets the answer, not
// a connection cut while it was still sending.
if received.is_err() {
use std::io::Read;
let rest = ledger_store::uploads::MAX_UPLOAD + 1;
let _ = std::io::copy(&mut (&mut body).take(rest), &mut std::io::sink());
}
let (status, outcome, text) = match &received {
Ok(g) => (
StatusCode::OK,
Outcome::Ok,
serde_json::json!({"received": g.id, "size": g.size, "sha256": g.sha256})
.to_string(),
),
Err(e) => {
let (status, code) = match e {
_ if stopped => (StatusCode::SERVICE_UNAVAILABLE, "stopped"),
UploadError::Unknown(_) => (StatusCode::NOT_FOUND, "no_such_upload"),
UploadError::NotYours => (StatusCode::FORBIDDEN, "not_yours"),
UploadError::AlreadySent => (StatusCode::CONFLICT, "already_sent"),
UploadError::InUse => (StatusCode::CONFLICT, "in_use"),
UploadError::WrongSize { .. }
| UploadError::TooMuch { .. }
| UploadError::WrongHash => (StatusCode::BAD_REQUEST, "not_as_declared"),
_ => (StatusCode::INTERNAL_SERVER_ERROR, "failed"),
};
(status, Outcome::Refused(code.into()), e.to_string())
}
};
let record = Access {
principal: Principal {
id: token.id.clone(),
kind: "agent".into(),
label: token.name.clone(),
},
client: None,
request_id: None,
action: "upload".into(),
subject: id.clone(),
args_digest: None,
result: outcome.clone(),
head_after: None,
};
let recorded = access.record(&record).is_ok();
if !recorded && received.is_ok() {
// Never bytes kept without their record: sent again later.
uploads.discard(&id);
}
(status, outcome, text, recorded)
});
let Ok((status, outcome, text, true)) = task.await else {
store.stop();
return (
StatusCode::SERVICE_UNAVAILABLE,
[(REFUSED_HEADER, "not-recorded")],
crate::access::NOT_RECORDED,
)
.into_response();
};
match outcome {
Outcome::Refused(code) => (status, [(REFUSED_HEADER, code)], text).into_response(),
_ => (status, text).into_response(),
}
}
/// `GET /b/{book}/a/{attachment}`: a stored file's bytes, for a token
/// with books:read (INVARIANT 9; what `wanderbooks download` asks for);
/// recorded in the access log (action `download`).
async fn download(
axum::extract::State(books): axum::extract::State<Arc<Books>>,
axum::extract::Path((book, attachment)): axum::extract::Path<(String, String)>,
request: axum::extract::Request,
) -> axum::response::Response {
use axum::http::StatusCode;
use axum::response::IntoResponse;
// GET only (axum hands a HEAD to a GET route): a HEAD would read the
// file and record a download that sent nothing.
if request.method() != axum::http::Method::GET {
return (
StatusCode::METHOD_NOT_ALLOWED,
[(axum::http::header::ALLOW, "GET")],
"a download is a GET",
)
.into_response();
}
let _counted = InFlight::new(&books);
books
.last_arrival
.store(books.started.elapsed().as_millis() as u64, Ordering::SeqCst);
let asked = Asked {
action: "download",
subject: attachment.clone(),
};
let (held, token) = match door(&books, &book, &asked, request.headers()).await {
Ok(passed) => passed,
Err(refused) => return *refused,
};
let Book::Open { access, store, .. } = &held else {
unreachable!("the door lets only open books through")
};
// The read and its record are one blocking task, run to its end even
// if the client hangs up, and counted until then.
let (access, store_for) = (access.clone(), store.clone());
let task = tokio::task::spawn_blocking(move || {
let _counted = _counted;
// An id as the books write it (no sign, no leading zeros), so the
// log holds no alias of it.
let id = attachment
.parse::<u64>()
.ok()
.filter(|n| n.to_string() == attachment);
let found = match (token.scopes.contains(&Scope::Read), id) {
// Stopped after an internal error: as for uploads.
_ if store_for.is_stopped() => Err((
StatusCode::SERVICE_UNAVAILABLE,
"stopped",
crate::server::STOPPED.to_string(),
)),
(false, _) => Err((
StatusCode::FORBIDDEN,
"insufficient_scope",
format!(
"this agent's token can't do that: a download needs books:read, and its token ({:?}) hasn't it",
token.name
),
)),
(true, None) => Err((
StatusCode::NOT_FOUND,
"no_such_attachment",
"there is no such attachment".to_string(),
)),
(true, Some(id)) => match store_for.attachment_file(id) {
Ok(Some(file)) => Ok(file),
Ok(None) => Err((
StatusCode::NOT_FOUND,
"no_such_attachment",
"there is no such attachment".to_string(),
)),
// Stopped, or the stored file damaged: not a missing one.
Err(why) => Err((StatusCode::INTERNAL_SERVER_ERROR, "failed", why)),
},
};
let record = Access {
principal: Principal {
id: token.id.clone(),
kind: "agent".into(),
label: token.name.clone(),
},
client: None,
request_id: None,
action: "download".into(),
subject: attachment.clone(),
args_digest: None,
result: match &found {
Ok(_) => Outcome::Ok,
Err((_, code, _)) => Outcome::Refused((*code).into()),
},
head_after: None,
};
let recorded = access.record(&record).is_ok();
(found, recorded)
});
let Ok((found, true)) = task.await else {
store.stop();
return (
StatusCode::SERVICE_UNAVAILABLE,
[(REFUSED_HEADER, "not-recorded")],
crate::access::NOT_RECORDED,
)
.into_response();
};
match found {
Ok((name, mime, bytes)) => {
let digest = format!(
"sha-256=:{}:",
base64::Engine::encode(
&base64::engine::general_purpose::STANDARD,
<sha2::Sha256 as sha2::Digest>::digest(&bytes)
)
);
let mime = axum::http::HeaderValue::from_str(&mime).unwrap_or(
axum::http::HeaderValue::from_static("application/octet-stream"),
);
(
StatusCode::OK,
[
(axum::http::header::CONTENT_TYPE, mime),
(
axum::http::header::CONTENT_DISPOSITION,
axum::http::HeaderValue::from_str(&disposition(&name))
.expect("only visible ASCII"),
),
(
axum::http::header::X_CONTENT_TYPE_OPTIONS,
axum::http::HeaderValue::from_static("nosniff"),
),
(
axum::http::HeaderName::from_static(DIGEST_HEADER),
axum::http::HeaderValue::from_str(&digest).expect("base64 is ASCII"),
),
],
bytes,
)
.into_response()
}
Err((status, code, why)) => (status, [(REFUSED_HEADER, code)], why).into_response(),
}
}
/// The header a download's SHA-256 rides in (RFC 9530 Repr-Digest), so
/// `wanderbooks download` keeps only what was stored.
pub const DIGEST_HEADER: &str = "repr-digest";
/// A download's Content-Disposition (RFC 6266): its name in ASCII, every
/// other character `_`, and whole in UTF-8 as `filename*` (RFC 8187).
fn disposition(name: &str) -> String {
let plain: String = name
.chars()
.map(|c| {
if (c.is_ascii_graphic() || c == ' ') && c != '"' && c != '\\' {
c
} else {
'_'
}
})
.collect();
let mut encoded = String::new();
for b in name.bytes() {
if b.is_ascii_alphanumeric() || b"!#$&+-.^_`|~".contains(&b) {
encoded.push(b as char);
} else {
encoded.push_str(&format!("%{b:02X}"));
}
}
format!("attachment; filename=\"{plain}\"; filename*=UTF-8''{encoded}")
}
/// `/b/{book}/mcp`: the book's MCP service, for a request carrying a token
/// for that book (INVARIANT 8).
async fn mcp(
axum::extract::State(books): axum::extract::State<Arc<Books>>,
axum::extract::Path(book): axum::extract::Path<String>,
mut request: axum::extract::Request,
) -> axum::response::Response {
use axum::http::StatusCode;
use tower::ServiceExt;
// Counted before the closing check, so closing never misses it.
let counted = (request.method() != axum::http::Method::GET).then(|| InFlight::new(&books));
books
.last_arrival
.store(books.started.elapsed().as_millis() as u64, Ordering::SeqCst);
let method = request.method().clone();
let (held, token) = match door(&books, &book, &Asked::connect(&method), request.headers()).await
{
Ok(passed) => passed,
Err(refused) => return *refused,
};
let Book::Open { service, .. } = &held else {
unreachable!("the door lets only open books through")
};
let service = service.clone();
let principal = |t: &Token| Principal {
id: t.id.clone(),
kind: "agent".into(),
label: t.name.clone(),
};
let session = request
.headers()
.get(SESSION_HEADER)
@@ -1024,7 +1389,7 @@ async fn mcp(
return refused_at_door(
&held,
principal(&token),
&method,
&Asked::connect(&method),
"too_many_sessions",
StatusCode::SERVICE_UNAVAILABLE,
"the books server has too many sessions open: end some, then try again".into(),
@@ -1035,7 +1400,7 @@ async fn mcp(
return refused_at_door(
&held,
principal(&token),
&method,
&Asked::connect(&method),
"session_token",
StatusCode::FORBIDDEN,
"this session was opened with another token".into(),
@@ -1154,7 +1519,7 @@ fn bind(run: &Path, name: &str) -> std::io::Result<tokio::net::UnixListener> {
}
fn print_usage(prog: &str) {
eprintln!("usage: {prog} [--root DIR] [--import-root DIR]...");
eprintln!("usage: {prog} [--root DIR]");
eprintln!(" The books server for the data root: started by the host, not by hand.");
eprintln!(" It runs until its standard input ends, or SIGTERM, SIGINT or SIGHUP.");
}
@@ -1166,14 +1531,10 @@ pub fn run(
args: impl IntoIterator<Item = String>,
) -> Result<(), Box<dyn std::error::Error>> {
let mut root: Option<PathBuf> = None;
let mut import_roots: Vec<PathBuf> = Vec::new();
let mut args = args.into_iter();
while let Some(arg) = args.next() {
match arg.as_str() {
"--root" => root = Some(PathBuf::from(args.next().ok_or("--root needs a folder")?)),
"--import-root" => import_roots.push(PathBuf::from(
args.next().ok_or("--import-root needs a folder")?,
)),
"--help" | "-h" => {
print_usage(prog);
return Ok(());
@@ -1240,14 +1601,8 @@ pub fn run(
std::fs::set_permissions(&lock_path, std::fs::Permissions::from_mode(0o600))
.map_err(|e| format!("can't make the server lock private: {}", e.kind()))?;
}
let import_roots = if import_roots.is_empty() {
ImportRoots::defaults()
} else {
ImportRoots::new(import_roots)
};
let books = Arc::new(Books {
root: root.clone(),
import_roots,
held: RwLock::new(BTreeMap::new()),
sessions: CancellationToken::new(),
closing: CancellationToken::new(),
@@ -1326,9 +1681,37 @@ async fn serve(
}
let held: Vec<String> = books.held().keys().cloned().collect();
eprintln!("wanderbooks server: ready, holding {} books", held.len());
// Every open book's lapsed uploads go within SWEEP_EVERY of lapsing,
// whatever else the server is doing (uploads.rs INVARIANT 3).
let (sweeping, sweep_stop) = (books.clone(), stop.clone());
tokio::spawn(async move {
let mut tick = tokio::time::interval(SWEEP_EVERY);
loop {
tokio::select! {
() = sweep_stop.cancelled() => break,
_ = tick.tick() => {
let uploads: Vec<_> = sweeping
.held()
.values()
.filter_map(|b| match b {
Book::Open { uploads, .. } => Some(uploads.clone()),
Book::Failed(_) => None,
})
.collect();
let now = chrono::Utc::now().timestamp();
let _ = tokio::task::spawn_blocking(move || {
uploads.iter().for_each(|u| u.sweep(now));
})
.await;
}
}
}
});
let app = axum::Router::new()
.route("/b/{book}/mcp", axum::routing::any(mcp))
.route("/b/{book}/u/{upload}", axum::routing::put(upload))
.route("/b/{book}/a/{attachment}", axum::routing::get(download))
.fallback(not_found)
.with_state(books.clone());
// The HTTP side keeps answering (refusing, once closing) until the
crates/ledger-mcp/src/local_files.rs
-1430
View File
File diff suppressed because it is too large. Load diff
crates/ledger-mcp/src/paths.rs
+132
View File
@@ -0,0 +1,132 @@
//! Paths as the server judges them: where a path really leads.
use std::path::{Component, Path, PathBuf};
/// `path` made absolute and resolved as the kernel would follow it, name
/// by name: every symlink followed (a dangling one too, to where it
/// points), `.` and `..` folded in order, and whatever doesn't exist kept
/// as written. For an existing path this is its canonical form; for a
/// missing one, where it would be.
pub fn resolved(path: &Path) -> PathBuf {
let abs = if path.is_absolute() {
path.to_path_buf()
} else {
std::env::current_dir()
.unwrap_or_else(|_| PathBuf::from("/"))
.join(path)
};
// The names still to walk, last first.
let mut todo: Vec<std::ffi::OsString> = Vec::new();
let mut out = PathBuf::new();
// An absolute path starts the walk over at its root.
let push_all = |todo: &mut Vec<std::ffi::OsString>, p: &Path, out: &mut PathBuf| {
*out = PathBuf::new();
let mut parts = Vec::new();
for c in p.components() {
match c {
Component::Prefix(_) | Component::RootDir => out.push(c.as_os_str()),
other => parts.push(other.as_os_str().to_os_string()),
}
}
todo.extend(parts.into_iter().rev());
};
push_all(&mut todo, &abs, &mut out);
// Links followed at most this many times (as the kernel's ELOOP).
let mut hops = 0;
while let Some(name) = todo.pop() {
if name == "." {
continue;
}
if name == ".." {
out.pop();
continue;
}
let next = out.join(&name);
let link = std::fs::symlink_metadata(&next)
.is_ok_and(|m| m.file_type().is_symlink())
.then(|| std::fs::read_link(&next).ok())
.flatten();
match link {
Some(target) if hops < 40 => {
hops += 1;
if target.is_absolute() {
push_all(&mut todo, &target, &mut out);
} else {
todo.extend(
target
.components()
.map(|c| c.as_os_str().to_os_string())
.rev(),
);
}
}
_ => out = next,
}
}
out
}
#[cfg(all(test, unix))]
mod tests {
use super::*;
use std::os::unix::fs::symlink;
/// `resolved` follows as many links as the kernel does (40, as
/// `canonicalize` does), and no more.
#[test]
fn resolved_follows_forty_links_and_no_more() {
let dir = tempfile::tempdir().unwrap();
let b = dir.path().canonicalize().unwrap();
let file = b.join("receipt.pdf");
std::fs::write(&file, "x").unwrap();
let link = |n: usize| b.join(format!("link-{n}"));
symlink(&file, link(1)).unwrap();
for n in 2..=41 {
symlink(link(n - 1), link(n)).unwrap();
}
assert_eq!(link(40).canonicalize().unwrap(), file);
assert_eq!(resolved(&link(40)), file);
assert!(link(41).canonicalize().is_err(), "the kernel stops at 40");
assert_ne!(resolved(&link(41)), file);
}
/// `resolved` places a path where the kernel would follow it: the
/// canonical form for one that exists, and through a dangling link to
/// where it points.
#[test]
fn resolved_follows_every_link_a_dangling_one_too() {
let dir = tempfile::tempdir().unwrap();
let b = dir.path().canonicalize().unwrap();
std::fs::create_dir_all(b.join("Downloads/sub")).unwrap();
std::fs::create_dir_all(b.join("data")).unwrap();
std::fs::write(b.join("Downloads/receipt.pdf"), "x").unwrap();
std::fs::write(b.join("data/journal.jsonl"), "x").unwrap();
symlink(
b.join("Downloads/receipt.pdf"),
b.join("Downloads/good-link"),
)
.unwrap();
symlink(b.join("data"), b.join("Downloads/data-link")).unwrap();
for p in [
b.join("Downloads/good-link"),
b.join("Downloads/data-link/journal.jsonl"),
b.join("Downloads/sub/../receipt.pdf"),
b.join("Downloads/./good-link"),
] {
assert_eq!(resolved(&p), p.canonicalize().unwrap(), "{}", p.display());
}
symlink("../elsewhere/gone.pdf", b.join("Downloads/rel-dangling")).unwrap();
assert_eq!(
resolved(&b.join("Downloads/rel-dangling")),
b.join("elsewhere/gone.pdf")
);
symlink(b.join("Downloads/loop-b"), b.join("Downloads/loop-a")).unwrap();
symlink(b.join("Downloads/loop-a"), b.join("Downloads/loop-b")).unwrap();
// A loop ends (the hop limit) inside the folder it loops in.
assert!(resolved(&b.join("Downloads/loop-a")).starts_with(b.join("Downloads")));
assert_eq!(
resolved(&b.join("Downloads/missing/../../elsewhere/x")),
b.join("elsewhere/x")
);
}
}
crates/ledger-mcp/src/scopes.rs
+1 -1
View File
@@ -119,7 +119,7 @@ pub const TABLE: [(&str, Needs); 109] = [
("books.assign_role", P),
("books.set_tax_category", W),
("books.import_chart_csv", W),
("books.import_local_file", W),
("books.begin_upload", W),
("books.export", R),
("books.register_asset", W),
("books.asset_schedule", R),
crates/ledger-mcp/src/server.rs
+374 -268
View File
@@ -32,7 +32,7 @@
//! ```
use crate::handbook;
use crate::local_files::{ImportRoots, Places, Protected, resolved};
use crate::paths::resolved;
use crate::policy::ReceiptPolicy;
use crate::settings::{Live, Settings};
use crate::tin_files;
@@ -57,9 +57,6 @@ use rmcp::{ServerHandler, tool, tool_handler, tool_router};
use rust_decimal::Decimal;
use std::sync::{Arc, Mutex};
/// Max receipt/document upload: 25 MB.
const MAX_UPLOAD_BYTES: usize = 25 * 1024 * 1024;
/// The server. One store, one receipt policy, shared across tool calls.
/// `Store` is `!Sync`, so access is serialized through a mutex; the server
/// is single-writer by construction. Tool bodies (synchronous disk and
@@ -69,12 +66,14 @@ const MAX_UPLOAD_BYTES: usize = 25 * 1024 * 1024;
pub struct BooksServer {
store: Arc<SharedStore>,
settings: Arc<Mutex<Live>>,
import_roots: Arc<ImportRoots>,
layout: Arc<Layout>,
/// The book's access log, for books served from the data root: every
/// call is then checked against its caller's token and recorded
/// (access.rs). None for a books dir served over stdio.
access: Option<Arc<ledger_store::access::AccessLog>>,
/// The book's uploads (books served from the data root; none over
/// stdio, where no bytes can be sent but inline).
uploads: Option<ledger_store::uploads::Uploads>,
}
/// Where a book's own files live around its data dir.
@@ -157,6 +156,31 @@ pub struct SharedStore {
}
impl SharedStore {
/// An attachment's name, type and bytes (hash-verified), for a
/// download: blocking (the store's lock, then the file).
/// None when there is no such attachment.
pub fn attachment_file(&self, id: u64) -> Result<Option<(String, String, Vec<u8>)>, String> {
if self.stopped() {
return Err(STOPPED.to_string());
}
let store = self.store.blocking_lock();
let Some(meta) = store
.list_attachments(None)
.map_err(|e| e.to_string())?
.into_iter()
.find(|a| a.id == id)
else {
return Ok(None);
};
let bytes = store.attachment_bytes(id).map_err(|e| e.to_string())?;
Ok(Some((meta.file_name, meta.mime_type, bytes)))
}
/// Whether it stopped taking requests (INVARIANT 2).
pub fn is_stopped(&self) -> bool {
self.stopped()
}
/// Share an open store.
pub fn new(store: Store) -> Arc<Self> {
Arc::new(Self {
@@ -200,12 +224,29 @@ impl SharedStore {
}
}
/// Why a file with full TINs isn't written into an import root.
const TIN_FILE_IN_ROOT: &str = "a file with full TINs is never saved in the server's import folders (by default your downloads folder), where it could be filed back into the books: save it elsewhere, such as your documents folder";
/// A filed file's name and type: those given, else the upload's own.
fn named(
file_name: Option<String>,
mime_type: Option<String>,
file: &files::TakenFile,
) -> Result<(String, String), String> {
let name = file_name
.or_else(|| file.file_name.clone())
.ok_or("file_name is needed")?;
let mime = mime_type
.or_else(|| file.mime.clone())
.ok_or("mime_type is needed")?;
if let Some(why) = ledger_store::uploads::file_name_problem(&name)
.or_else(|| ledger_store::uploads::mime_problem(&mime))
{
return Err(why);
}
Ok((name, mime))
}
/// What every request gets after an internal error (SharedStore
/// INVARIANT 2).
const STOPPED: &str = "the books server stopped taking requests after an internal error, to protect the books: ask the human to restart the books server (the books are checked again when it starts)";
pub(crate) const STOPPED: &str = "the books server stopped taking requests after an internal error, to protect the books: ask the human to restart the books server (the books are checked again when it starts)";
/// Why the vault won't open with a key file inside the books.
const KEY_IN_BOOKS: &str = "the TIN vault's key file is inside the books' folder, beside the records it unlocks: ask the human to move it out of the books and set [vault] key_file in the settings";
@@ -555,12 +596,18 @@ impl BooksServer {
policy,
..Settings::default()
}))),
import_roots: Arc::new(ImportRoots::defaults()),
layout: Arc::new(Layout::BooksDir),
access: None,
uploads: None,
}
}
/// Take uploaded files from `uploads` (`books.begin_upload`).
pub fn with_uploads(mut self, uploads: ledger_store::uploads::Uploads) -> Self {
self.uploads = Some(uploads);
self
}
/// Check every call against its caller's token, and record it in
/// `log` (access.rs).
pub fn with_access(mut self, log: Arc<ledger_store::access::AccessLog>) -> Self {
@@ -575,12 +622,6 @@ impl BooksServer {
self
}
/// The folders `books.import_local_file` may read from (local_files.rs).
pub fn with_import_roots(mut self, roots: ImportRoots) -> Self {
self.import_roots = Arc::new(roots);
self
}
/// Take the settings from files, reloading them whenever they change
/// (crate::settings), shared with the other sessions holding `live`.
pub fn with_live_settings(mut self, live: Arc<Mutex<Live>>) -> Self {
@@ -655,7 +696,7 @@ impl BooksServer {
/// Every tool the server serves.
pub(crate) fn tool_router() -> rmcp::handler::server::router::tool::ToolRouter<Self> {
Self::core_router() + Self::kept_router() + Self::access_router()
Self::core_router() + Self::kept_router() + Self::access_router() + Self::files_router()
}
/// The settings in force for this request (reloaded at its start, in
@@ -902,20 +943,8 @@ impl BooksServer {
self.layout.vault_dir(store)
}
/// What is never read through a path, even inside an import root
/// (local_files.rs INVARIANT 3; Protected::of): the data dir, the
/// vault, the key file in force and the default one (a key left there
/// from before still unlocks the vault), and the folders it seals; and
/// the rest of the books dir.
fn protected(&self, store: &Store) -> Protected {
Protected::of(
&places(store, &self.settings(), &self.layout),
&self.import_roots.usable(),
)
}
/// Refuse bytes on their way into the books (books.attach_receipt,
/// books.file_document, books.import_local_file) if they carry a full
/// books.file_document, books.set_w9's file) if they carry a full
/// TIN: a file the server read or wrote with TINs, or text holding one
/// the vault holds or one printed as an SSN or EIN is (tin_files.rs
/// INVARIANT 3), or text probing for one (INVARIANT 4).
@@ -963,62 +992,78 @@ impl BooksServer {
.collect()
}
/// Read a file an agent names (books.import_local_file, books.set_w9
/// w9_file_path) through the import roots' lockdown (local_files.rs),
/// with its canonical path.
fn read_local(&self, path: &str) -> Result<(std::path::PathBuf, Vec<u8>), String> {
let protected = {
let store = self.lock()?;
self.protected(&store)
};
self.import_roots
.read(path.trim(), &protected, MAX_UPLOAD_BYTES)
}
/// Where a file holding full TINs may be written: a new file outside
/// the books (outside_books) and outside every import root, so it can
/// never be filed back in as a receipt and read out again.
/// the books (outside_books). (It can't be filed back in unnoticed:
/// tin_files.rs records it.)
fn tin_file_dest(&self, store: &Store, path: &str) -> Result<std::path::PathBuf, String> {
let dest = outside_books(&self.layout.books_dir(store), path)?;
self.outside_import_roots(&dest)?;
Ok(dest)
outside_books(&self.layout.books_dir(store), path)
}
fn outside_import_roots(&self, dest: &std::path::Path) -> Result<(), String> {
if self.import_roots.contains(dest) {
return Err(TIN_FILE_IN_ROOT.to_string());
}
Ok(())
}
/// Read a W-9 file (PDF, PNG, or JPEG, 25 MB at most) for the vault,
/// through the import roots' lockdown like any local file: the type
/// is judged by where the path really leads (a link named w9.pdf to
/// another file is that file).
fn read_w9_file(&self, path: &str) -> Result<ledger_vault::W9File, String> {
let (canonical, bytes) = self.read_local(path)?;
/// The signed W-9 a call gives (an upload, or inline), for the vault:
/// a PDF, PNG, or JPEG by its own first bytes, whatever it is called.
fn w9_file(
&self,
p: &SetW9Params,
) -> Result<(ledger_vault::W9File, Option<ledger_store::uploads::Claim>), String> {
let file = self.take_file(files::Given {
upload_id: p.w9_upload_id.as_deref(),
content_base64: p.w9_content_base64.as_deref(),
sha256: p.w9_sha256.as_deref(),
purposes: files::W9,
})?;
// Not recorded here: set_w9 records it once the call can only
// succeed, just before the vault write (tin_files.rs INVARIANT 1),
// so a refused call never marks an ordinary file.
let mime_type = match canonical
.extension()
.and_then(|e| e.to_str())
.map(str::to_lowercase)
.as_deref()
{
Some("pdf") => "application/pdf",
Some("png") => "image/png",
Some("jpg" | "jpeg") => "image/jpeg",
_ => return Err("w9_file_path is a PDF, PNG, or JPEG (.pdf, .png, .jpg)".into()),
let (mime_type, ext) = if file.bytes.starts_with(b"%PDF-") {
("application/pdf", "pdf")
} else if file.bytes.starts_with(b"\x89PNG\r\n\x1a\n") {
("image/png", "png")
} else if file.bytes.starts_with(&[0xFF, 0xD8, 0xFF]) {
("image/jpeg", "jpg")
} else {
return Err("the W-9 is a PDF, PNG, or JPEG".into());
};
Ok(ledger_vault::W9File {
file_name: canonical
.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_else(|| "w9".into()),
mime_type: mime_type.to_string(),
bytes,
})
let file_name = p
.w9_file_name
.clone()
.or(file.file_name)
.unwrap_or_else(|| format!("w9.{ext}"));
if let Some(why) = ledger_store::uploads::file_name_problem(&file_name) {
return Err(why);
}
Ok((
ledger_vault::W9File {
file_name,
mime_type: mime_type.to_string(),
bytes: file.bytes,
},
file.claim,
))
}
/// A CSV's text, given inline or as an upload (books.begin_upload),
/// and the upload's hold.
fn csv_given(
&self,
text: Option<&str>,
upload_id: Option<&str>,
) -> Result<(String, Option<ledger_store::uploads::Claim>), String> {
match (text, upload_id) {
(Some(_), Some(_)) => Err("give csv_text or upload_id, not both".into()),
(Some(t), None) => Ok((t.to_string(), None)),
(None, Some(id)) => {
let file = self.take_file(files::Given {
upload_id: Some(id),
content_base64: None,
sha256: None,
purposes: files::IMPORTED,
})?;
let text = String::from_utf8(file.bytes)
.map_err(|_| "the CSV isn't UTF-8 text".to_string())?;
Ok((text, file.claim))
}
(None, None) => Err("give the CSV: csv_text, or an upload_id".into()),
}
}
/// Record a file read or written with full TINs (tin_files.rs
@@ -1920,56 +1965,6 @@ fn known_tin_last4(store: &Store) -> Vec<u32> {
.collect()
}
/// Where these books keep what an import must never read
/// (local_files.rs): the data dir, the books dir, the vault, the key file
/// in force and the default one, and the home folder.
fn places(store: &Store, settings: &Settings, layout: &Layout) -> Places {
Places {
data_dir: store.data_dir().to_path_buf(),
books_dir: layout.books_dir(store),
vault_dir: layout.vault_dir(store),
key_files: vec![
layout.key_file(store, settings),
settings.key_file(),
ledger_vault::default_key_file(),
],
// In the data root the home comes from the account's record, as
// the root's own does; never only from the environment.
home: match layout {
Layout::Root(_) => ledger_store::root::account_home().ok(),
Layout::BooksDir => None,
}
.or_else(|| std::env::var_os("HOME").map(std::path::PathBuf::from)),
}
}
/// The import roots a server for these books may use (local_files.rs
/// INVARIANT 0): a named root that is `/`, the home folder or holds it,
/// or holds the books, the vault or a key, is refused (the server doesn't
/// start); a default one is dropped, leaving none. The text is for the
/// human starting the server, on its standard error.
pub fn guard_import_roots(
store: &Store,
settings: &Settings,
layout: &Layout,
roots: ImportRoots,
) -> Result<ImportRoots, String> {
let places = places(store, settings, layout);
let explicit = roots.explicit();
let (roots, refused) = roots.guard(places.home.as_deref(), &places.keep_out());
if let Some((root, why)) = refused.first() {
let msg = format!(
"import folder {} can't be used: {why}. Name a folder that holds none of them, such as your downloads folder",
root.display()
);
if explicit {
return Err(msg);
}
eprintln!("ledger-mcp: {msg}; books.import_local_file has no folder to read from");
}
Ok(roots)
}
/// Whether `path` is `dir` or inside it.
fn is_inside(dir: &std::path::Path, path: &std::path::Path) -> bool {
resolved(path).starts_with(resolved(dir))
@@ -2719,6 +2714,10 @@ fn movement_in(ledger: &Ledger, account: AccountId, from: NaiveDate, to: NaiveDa
}
thread_local! {
/// The token of the request running on this thread (books served from
/// the data root), its id and name, for what is bound to it (its
/// uploads) and the commands it is given.
static CALLER: std::cell::RefCell<Option<(String, String)>> = const { std::cell::RefCell::new(None) };
/// The journal's head after the last write made by the request
/// running on this thread (StoreGuard's drop), for its access record.
static WROTE: std::cell::RefCell<Option<(u64, String)>> = const { std::cell::RefCell::new(None) };
@@ -2892,6 +2891,8 @@ fn lines_out(lines: &[ledger_core::StatementLine]) -> Vec<StatementLineOut> {
#[path = "tools/access.rs"]
mod access_tools;
#[path = "tools/files.rs"]
mod files;
#[path = "tools/kept.rs"]
mod kept;
pub use kept::SettingKeyOut;
@@ -3548,7 +3549,7 @@ impl BooksServer {
#[tool(
name = "books.attach_receipt",
title = "Attach a receipt",
description = "Attach a receipt (base64 file bytes) to an entry, or file it unlinked and link later with books.link_receipt. Files are content-addressed and deduplicated. link_to links it to more subjects in the same call (another entry it also supports, an asset, a party, ...: see books.link_attachment); every subject is checked first, so one bad subject files nothing. A file holding a full TIN is refused: one the server read or wrote with TINs, or text with a TIN the vault keeps or an SSN or EIN printed 3-2-4 or 2-7 with hyphens.",
description = "Attach a receipt to an entry, or file it unlinked and link later with books.link_receipt. The bytes: upload_id (books.begin_upload), or content_base64 with its sha256 for a file of at most 256 KiB. Files are content-addressed and deduplicated. link_to links it to more subjects in the same call (another entry it also supports, an asset, a party, ...: see books.link_attachment); every subject is checked first, so one bad subject files nothing. A file holding a full TIN is refused: one the server read or wrote with TINs, or text with a TIN the vault keeps or an SSN or EIN printed 3-2-4 or 2-7 with hyphens.",
annotations(
read_only_hint = false,
destructive_hint = false,
@@ -3560,17 +3561,6 @@ impl BooksServer {
params: Parameters<AttachReceiptParams>,
) -> Result<Json<AttachReceiptOut>, String> {
let p = params.0;
let bytes = base64::Engine::decode(
&base64::engine::general_purpose::STANDARD,
&p.content_base64,
)
.map_err(|_| "content_base64 is not valid base64".to_string())?;
if bytes.len() > MAX_UPLOAD_BYTES {
return Err(format!(
"file is {} bytes; the limit is {MAX_UPLOAD_BYTES}",
bytes.len()
));
}
let mut store = self.lock()?;
if let Some(e) = p.entry_id
&& store.ledger().get_entry(e).is_none()
@@ -3578,16 +3568,26 @@ impl BooksServer {
return Err(format!("unknown entry {e}"));
}
let link_to = check_link_to(&store, p.entry_id, &p.link_to)?;
self.refuse_tin_file(&store, &bytes, &p.mime_type, &p.file_name)?;
// Taken once everything else is known good.
let file = self.take_file(files::Given {
upload_id: p.upload_id.as_deref(),
content_base64: p.content_base64.as_deref(),
sha256: p.sha256.as_deref(),
purposes: files::FILED,
})?;
let (file_name, mime_type) = named(p.file_name, p.mime_type, &file)?;
let (bytes, claim) = (file.bytes, file.claim);
self.refuse_tin_file(&store, &bytes, &mime_type, &file_name)?;
let (attachment_id, sha256, deduplicated) = store
.attach_receipt(
p.entry_id,
&p.file_name,
&p.mime_type,
&file_name,
&mime_type,
&bytes,
p.description.as_deref(),
)
.map_err(|e| e.to_string())?;
files::filed(claim);
for sub in link_to {
store
.link_attachment(attachment_id, sub)
@@ -3604,7 +3604,7 @@ impl BooksServer {
#[tool(
name = "books.file_document",
title = "File a document",
description = "File a document into the content-addressed store: category formation, legal, tax, accounting, statement (a bank or card statement: then books.record_statement), contract, photo, warranty, import_source (a bank export), or other. link_to links it to subjects in the same call (an account, an asset, a party, the company, ...: see books.link_attachment); every subject is checked first, so one bad subject files nothing. Never file a W-9 here (it shows the full TIN): books.set_w9 w9_file_path puts it in the vault. A file holding a full TIN is refused: one the server read or wrote with TINs, or text with a TIN the vault keeps or an SSN or EIN printed 3-2-4 or 2-7 with hyphens.",
description = "File a document: category formation, legal, tax, accounting, statement (a bank or card statement: then books.record_statement), contract, photo, warranty, import_source (a bank export), or other. link_to links it to subjects in the same call (an account, an asset, a party, the company: see books.link_attachment), all checked first. The bytes: upload_id (books.begin_upload), or content_base64 with its sha256 (at most 256 KiB). Never a W-9 (books.set_w9 w9_upload_id puts it in the vault). A file holding a full TIN is refused: one the server read or wrote with TINs, or text with a TIN the vault keeps or an SSN or EIN printed 3-2-4 or 2-7 with hyphens.",
annotations(
read_only_hint = false,
destructive_hint = false,
@@ -3626,29 +3626,27 @@ impl BooksServer {
));
}
}
let bytes = base64::Engine::decode(
&base64::engine::general_purpose::STANDARD,
&p.content_base64,
)
.map_err(|_| "content_base64 is not valid base64".to_string())?;
if bytes.len() > MAX_UPLOAD_BYTES {
return Err(format!(
"file is {} bytes; the limit is {MAX_UPLOAD_BYTES}",
bytes.len()
));
}
let mut store = self.lock()?;
let link_to = check_link_to(&store, None, &p.link_to)?;
self.refuse_tin_file(&store, &bytes, &p.mime_type, &p.file_name)?;
let file = self.take_file(files::Given {
upload_id: p.upload_id.as_deref(),
content_base64: p.content_base64.as_deref(),
sha256: p.sha256.as_deref(),
purposes: files::FILED,
})?;
let (file_name, mime_type) = named(p.file_name, p.mime_type, &file)?;
let (bytes, claim) = (file.bytes, file.claim);
self.refuse_tin_file(&store, &bytes, &mime_type, &file_name)?;
let (attachment_id, sha256, deduplicated) = store
.file_document(
&p.category,
&p.file_name,
&p.mime_type,
&file_name,
&mime_type,
&bytes,
p.description.as_deref(),
)
.map_err(|e| e.to_string())?;
files::filed(claim);
for sub in link_to {
store
.link_attachment(attachment_id, sub)
@@ -3924,7 +3922,7 @@ impl BooksServer {
#[tool(
name = "books.get_receipt",
title = "Get a receipt",
description = "Fetch a receipt or document's metadata (with everything it is linked to) and base64 bytes. Bytes are hash-verified on read.",
description = "A receipt or document: what it is and everything it is linked to. view bytes adds its bytes, base64 and hash-verified, for a file of at most 256 KiB; a larger one gives the command (wanderbooks download) that saves a copy in a folder you name (books served by `wanderbooks mcp`; elsewhere a larger file doesn't come back).",
annotations(read_only_hint = true, open_world_hint = false)
)]
async fn get_receipt(
@@ -3932,12 +3930,54 @@ impl BooksServer {
params: Parameters<GetReceiptParams>,
) -> Result<Json<GetReceiptOut>, String> {
let p = params.0;
let bytes_wanted = match p.view.as_deref().unwrap_or("meta") {
"meta" => false,
"bytes" => true,
other => {
return Err(format!(
"view {:?}: meta (the default) or bytes",
other.chars().take(20).collect::<String>()
));
}
};
let store = self.lock()?;
let meta = find_attachment(&store, p.attachment_id)?;
let bytes = store
.attachment_bytes(p.attachment_id)
.map_err(|e| e.to_string())?;
let book = store
.data_dir()
.file_name()
.map(|n| n.to_string_lossy().to_string())
.unwrap_or_default();
let fits = meta.size_bytes as usize <= files::INLINE_MAX;
let content_base64 = if bytes_wanted && fits {
let bytes = store
.attachment_bytes(p.attachment_id)
.map_err(|e| e.to_string())?;
Some(base64::Engine::encode(
&base64::engine::general_purpose::STANDARD,
&bytes,
))
} else {
None
};
// Too large to come back here: the command that saves a copy (books
// served from the data root; over stdio there is none).
let download = (bytes_wanted && !fits && self.uploads.is_some()).then(|| {
format!(
"wanderbooks download --book {book}{} {} --to <a folder>",
files::token_flag(),
meta.id
)
});
if bytes_wanted && !fits && download.is_none() {
return Err(format!(
"attachment {} is {} bytes; at most {} come back on this connection (a larger file needs these books served by `wanderbooks mcp`)",
meta.id,
meta.size_bytes,
files::INLINE_MAX
));
}
Ok(Json(GetReceiptOut {
download,
links: links_out(&store, meta.id)?,
description: meta.description.clone(),
attachment_id: meta.id,
@@ -3947,10 +3987,7 @@ impl BooksServer {
file_name: meta.file_name,
mime_type: meta.mime_type,
size_bytes: meta.size_bytes,
content_base64: base64::Engine::encode(
&base64::engine::general_purpose::STANDARD,
&bytes,
),
content_base64,
}))
}
@@ -4413,7 +4450,7 @@ impl BooksServer {
#[tool(
name = "books.import_bank_csv",
title = "Import a bank CSV as drafts",
description = "Import a QuickBooks-style bank/credit-card CSV: 3 columns Date,Description,Amount (payments negative), or 4 columns Date,Description, money out, money in (Intuit heads them Credit,Debit). Slash dates must settle month-first or day-first from the file itself or from date_order (\"month_first\" for mm/dd, \"day_first\" for dd/mm); ask the human when the tool says the dates read either way. Creates one DRAFT entry per transaction, all or none: bank account on one side, categorized offset on the other (the longest matching rule pattern, else default_offset_code). Review with books.digest, then post. File the export first (books.file_document category import_source) and pass source_attachment_id: it is linked to the bank account, so the import traces to its source (never to the drafts: a bank export is not a receipt).",
description = "Import a QuickBooks-style bank/credit-card CSV: 3 columns Date,Description,Amount (payments negative), or 4 columns Date,Description, money out, money in (Intuit heads them Credit,Debit). Slash dates must settle month-first or day-first from the file itself or from date_order (\"month_first\" for mm/dd, \"day_first\" for dd/mm); ask the human when the tool says the dates read either way. Creates one DRAFT entry per transaction, all or none: bank account on one side, categorized offset on the other (the longest matching rule pattern, else default_offset_code). Give the CSV as csv_text, or upload it (books.begin_upload purpose import) and give upload_id. Review with books.digest, then post. File the export too (books.file_document category import_source) and pass source_attachment_id: it is linked to the bank account, so the import traces to its source (never to the drafts: a bank export is not a receipt).",
annotations(
read_only_hint = false,
destructive_hint = false,
@@ -4435,8 +4472,8 @@ impl BooksServer {
));
}
};
let txns =
ledger_import::parse_bank_csv_with(&p.csv_text, order).map_err(|e| e.to_string())?;
let (csv, claim) = self.csv_given(p.csv_text.as_deref(), p.upload_id.as_deref())?;
let txns = ledger_import::parse_bank_csv_with(&csv, order).map_err(|e| e.to_string())?;
let mut store = self.lock()?;
let bank = store
.ledger()
@@ -4507,6 +4544,8 @@ impl BooksServer {
.map_err(|e| format!("{} {}: {e}", txn.date, txn.description));
e?;
}
// Rehearsed whole: the upload is used up as the writing begins.
files::filed(claim);
let mut imported = Vec::new();
for txn in txns {
let (offset_id, offset_code) = offset_for(&txn.description);
@@ -5578,7 +5617,7 @@ impl BooksServer {
#[tool(
name = "books.import_chart_csv",
title = "Import a chart of accounts",
description = "Import a chart of accounts from CSV (header: code,name,type,parent_code). Types: asset, liability, equity, revenue, expense. Parents must come before children. Fails on the first bad row; nothing is half-imported.",
description = "Import a chart of accounts from CSV (header: code,name,type,parent_code): csv_text, or upload_id (books.begin_upload purpose import). Types: asset, liability, equity, revenue, expense. Parents must come before children. Fails on the first bad row; nothing is half-imported.",
annotations(
read_only_hint = false,
destructive_hint = false,
@@ -5590,7 +5629,8 @@ impl BooksServer {
params: Parameters<ImportChartCsvParams>,
) -> Result<Json<ImportChartCsvOut>, String> {
let p = params.0;
let rows = ledger_import::parse_chart_csv(&p.csv_text).map_err(|e| e.to_string())?;
let (csv, claim) = self.csv_given(p.csv_text.as_deref(), p.upload_id.as_deref())?;
let rows = ledger_import::parse_chart_csv(&csv).map_err(|e| e.to_string())?;
let mut store = self.lock()?;
let parse_type = |t: &str| match t {
"asset" => Ok(AccountType::Asset),
@@ -5633,6 +5673,8 @@ impl BooksServer {
.map_err(|e| format!("{}: {e}", row.code))?;
}
}
// Rehearsed whole: the upload is used up as the writing begins.
files::filed(claim);
let mut created = Vec::new();
for row in rows {
let account_type = match row.account_type.as_str() {
@@ -5668,71 +5710,6 @@ impl BooksServer {
Ok(Json(ImportChartCsvOut { created, count }))
}
#[tool(
name = "books.import_local_file",
title = "File a local file",
description = "File a receipt or document from a file on this computer: an absolute path to a regular file in the server's import folders (by default your downloads folder; save it there first). Files of the books themselves are never read. A file the server read or wrote with full TINs (a W-9, a 1099 worksheet, a CPA package or a file from one), and text holding a TIN the vault keeps or a number printed as an SSN or EIN is (digits 3-2-4 or 2-7 joined by hyphens), are refused: they never go into the books.",
annotations(
read_only_hint = false,
destructive_hint = false,
open_world_hint = false
)
)]
async fn import_local_file(
&self,
params: Parameters<ImportLocalFileParams>,
) -> Result<Json<ImportLocalFileOut>, String> {
let p = params.0;
// The name and type are those of the file read, where the path
// really leads (a link named receipt.pdf to a CSV is a CSV).
let (canonical, bytes) = self.read_local(&p.path)?;
let file_name = p.file_name.unwrap_or_else(|| {
canonical
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("file")
.to_string()
});
let mime_type = mime_guess::from_path(&canonical)
.first_or_octet_stream()
.to_string();
let mut store = self.lock()?;
// A file the server read or wrote with full TINs, or text holding
// one (the vault's, or one printed as an SSN or EIN is), never goes
// into the books (tin_files.rs).
self.refuse_tin_file(&store, &bytes, &mime_type, &file_name)?;
let (attachment_id, sha256, deduplicated) = match p.category.as_str() {
"receipt" => store
.attach_receipt(
p.entry_id,
&file_name,
&mime_type,
&bytes,
p.description.as_deref(),
)
.map_err(|e| e.to_string())?,
"document" => {
let cat = p.document_category.as_deref().unwrap_or("other");
store
.file_document(
cat,
&file_name,
&mime_type,
&bytes,
p.description.as_deref(),
)
.map_err(|e| e.to_string())?
}
_ => return Err("category must be \"receipt\" or \"document\"".to_string()),
};
Ok(Json(ImportLocalFileOut {
attachment_id,
sha256,
deduplicated,
file_name,
}))
}
#[tool(
name = "books.export",
title = "Export a report",
@@ -7097,7 +7074,7 @@ impl BooksServer {
#[tool(
name = "books.set_w9",
title = "Record W-9 facts",
description = "Record what a contractor's or landlord's W-9 says, or clear it (clear: true alone): the date received (received_on), line 3 entity type, line 4 exempt payee and FATCA codes, the TIN's last four digits, and where the form is kept. Give the full TIN as tin with tin_type (ssn, ein, itin) and it goes only to the encrypted TIN vault (its key kept outside the books, created on first use): the books keep its last four (tin_last4 is then optional, and refused if it disagrees) and a hash of the vault record, in the W-9 facts and the party's TIN facts alike (party.tin_set, as books.set_tin), and kept_at defaults to \"wanderbooks vault\". The CPA package carries the full TIN to the CPA. Give w9_file_path (a local PDF, PNG, or JPEG of the signed form) and the form goes to the vault too, never to the receipts and documents; read it back with books.get_w9. Never put a TIN in any other field, note, description, or file: the engine refuses any nine-digit number, and no tool output echoes one. The party needs a 1099 role (contractor or landlord: books.set_party_roles). Record it before paying them. A party with a vendor record also gets its W-9 marked on file there (books.vendor_1099 missing_w9). Clearing W-9 facts whose TIN is in the vault needs books.purge_tin first. Journaled as party.w9_set. Returns the party (the TIN's last four only).",
description = "Record what a contractor's or landlord's W-9 says, or clear it (clear: true alone): the date received (received_on), line 3 entity type, line 4 exempt payee and FATCA codes, the TIN's last four digits, and where the form is kept. Give the full TIN as tin with tin_type (ssn, ein, itin) and it goes only to the encrypted TIN vault (its key kept outside the books, created on first use): the books keep its last four (tin_last4 is then optional, and refused if it disagrees) and a hash of the vault record, in the W-9 facts and the party's TIN facts alike (party.tin_set, as books.set_tin), and kept_at defaults to \"wanderbooks vault\". The CPA package carries the full TIN to the CPA. Give the signed form (a PDF, PNG, or JPEG) as w9_upload_id (books.begin_upload), or w9_content_base64 with w9_sha256 when at most 256 KiB, and it goes to the vault too, never to the receipts and documents; read it back with books.get_w9. Never put a TIN in any other field, note, description, or file: the engine refuses any nine-digit number, and no tool output echoes one. The party needs a 1099 role (contractor or landlord: books.set_party_roles). Record it before paying them. A party with a vendor record also gets its W-9 marked on file there (books.vendor_1099 missing_w9). Clearing W-9 facts whose TIN is in the vault needs books.purge_tin first. Journaled as party.w9_set. Returns the party (the TIN's last four only).",
annotations(
read_only_hint = false,
destructive_hint = true,
@@ -7106,7 +7083,14 @@ impl BooksServer {
)]
async fn set_w9(&self, params: Parameters<SetW9Params>) -> Result<Json<PartyOut>, String> {
let p = params.0;
let vault_input = p.tin.is_some() || p.tin_type.is_some() || p.w9_file_path.is_some();
let w9_given = p.w9_upload_id.is_some() || p.w9_content_base64.is_some();
if p.w9_file_name.is_some() && !w9_given {
return Err(
"w9_file_name names the W-9 given with it (w9_upload_id or w9_content_base64)"
.into(),
);
}
let vault_input = p.tin.is_some() || p.tin_type.is_some() || w9_given;
let tin = match (&p.tin, &p.tin_type) {
(Some(t), Some(k)) => Some(
ledger_vault::Tin::new(t.0.trim(), parse_tin_type(k)?)
@@ -7159,11 +7143,12 @@ impl BooksServer {
vault_record: None,
})
};
let w9_file = p
.w9_file_path
.as_deref()
.map(|path| self.read_w9_file(path))
.transpose()?;
let (w9_file, mut w9_claim) = if w9_given {
let (file, claim) = self.w9_file(&p)?;
(Some(file), claim)
} else {
(None, None)
};
let mut store = self.lock()?;
let id = resolve_party(&store, &p.party)?;
let existing = store.ledger().get_party(id).and_then(|p| p.w9.clone());
@@ -7245,6 +7230,8 @@ impl BooksServer {
Some(w) => Some(vault.put_w9(id, w).map_err(|e| e.to_string())?),
None => None,
};
// The W-9 is in the vault: its upload is used up.
files::filed(w9_claim.take());
match tin_hash {
Some(h) => Some(h),
None if held_tin => vault.record_hash(id).map_err(|e| e.to_string())?,
@@ -7423,7 +7410,7 @@ impl BooksServer {
.map_err(|e| e.to_string())?
.ok_or_else(|| {
format!(
"the TIN vault holds no W-9 file for party {id}: store one with books.set_w9 w9_file_path"
"the TIN vault holds no W-9 file for party {id}: store one with books.set_w9 w9_upload_id"
)
})?;
self.wrote_tins(&store, &file.bytes)?;
@@ -8500,11 +8487,6 @@ impl BooksServer {
)
})
.transpose()?;
if p.include_tins
&& let Some(d) = &dest
{
self.outside_import_roots(d)?;
}
let meta = self.report_meta(&store);
let ledger = store.ledger();
// The vault, opened once and never creating a key: locked, each
@@ -9588,7 +9570,10 @@ impl BooksServer {
.parse()
.map_err(|_| format!("bad receipt resource URI: {uri}"))?;
let Json(out) = self
.get_receipt(Parameters(GetReceiptParams { attachment_id: id }))
.get_receipt(Parameters(GetReceiptParams {
attachment_id: id,
view: None,
}))
.await?;
serde_json::to_string_pretty(&out).map_err(|e| e.to_string())
}
@@ -9703,6 +9688,11 @@ impl ServerHandler for BooksServer {
let result = run_blocking(&self.store, move || {
Box::pin(async move {
WROTE.with(|w| w.borrow_mut().take());
let caller = running_call.as_ref().and_then(|c| {
c.caller()
.map(|c| (c.token.id.clone(), c.token.name.clone()))
});
CALLER.with(|c| *c.borrow_mut() = caller);
// Counted as running by now (run_blocking): hashed here.
let digest = running_call
.as_ref()
@@ -10073,11 +10063,16 @@ mod tests {
}
}
/// "Max 25 MB" (books.attach_receipt): 25 MiB exactly.
/// An upload's limit, 25 MiB exactly; inline, 256 KiB (plan 2.1, 4.7).
#[test]
fn the_upload_limit_is_25_mib() {
assert_eq!(MAX_UPLOAD_BYTES, 25 * 1024 * 1024);
assert_eq!(MAX_UPLOAD_BYTES, 26_214_400);
fn the_file_limits_are_the_plans() {
assert_eq!(ledger_store::uploads::MAX_UPLOAD, 26_214_400);
assert_eq!(files::INLINE_MAX, 262_144);
// Plan 4.7: an upload lasts 15 minutes; 16 under way per token.
assert_eq!(ledger_store::uploads::UPLOAD_TTL, 900);
assert_eq!(ledger_store::uploads::MAX_LIVE, 16);
assert_eq!(ledger_store::uploads::MAX_NAME_BYTES, 255);
assert_eq!(crate::local::MAX_BODY, 1024 * 1024);
}
/// A period is "YYYY-MM": a year of the calendar the books keep
@@ -10450,7 +10445,7 @@ mod tests {
("books.assign_role", D),
("books.set_tax_category", W),
("books.import_chart_csv", W),
("books.import_local_file", W),
("books.begin_upload", W),
("books.export", R),
("books.register_asset", W),
("books.asset_schedule", R),
@@ -10577,6 +10572,10 @@ mod tests {
"ledger-import xlsx",
include_str!("../../ledger-import/src/xlsx.rs"),
),
(
"ledger-store uploads",
include_str!("../../ledger-store/src/uploads.rs"),
),
] {
let mut rest = src;
let mut n = 0;
@@ -10589,6 +10588,18 @@ mod tests {
}
assert!(n > 0, "{krate}: no error templates found");
}
// The files tool's refusals (and description): its sentences.
for line in include_str!("tools/files.rs").lines() {
let line = line.trim_start();
if line.starts_with("//") {
continue;
}
for (i, text) in line.split('"').enumerate() {
if i % 2 == 1 && text.len() > 30 && text.contains(' ') {
out.push(("tools/files.rs text".into(), text.to_string()));
}
}
}
let newer = |verified| ledger_store::StoreError::NewerJournal {
seq: 7,
event_type: "settings.set".into(),
@@ -10605,25 +10616,11 @@ mod tests {
));
for (what, text) in [
("STOPPED", STOPPED),
("TIN_FILE_IN_ROOT", TIN_FILE_IN_ROOT),
("KEY_IN_BOOKS", KEY_IN_BOOKS),
("VAULT_IN_HISTORY", VAULT_IN_HISTORY),
("INSIDE_BOOKS", INSIDE_BOOKS),
("NO_FOLDER", NO_FOLDER),
("EXISTS", EXISTS),
("local_files::CHANGED", crate::local_files::CHANGED),
(
"local_files::NO_SUCH_FILE",
crate::local_files::NO_SUCH_FILE,
),
("local_files::NO_ROOTS", crate::local_files::NO_ROOTS),
("local_files::SEALED", crate::local_files::SEALED),
(
"local_files::IN_THE_BOOKS",
crate::local_files::IN_THE_BOOKS,
),
("local_files::HARD_LINKED", crate::local_files::HARD_LINKED),
("local_files::NOT_REGULAR", crate::local_files::NOT_REGULAR),
("tin_files::TIN_FILE", tin_files::TIN_FILE),
("tin_files::TIN_HELD", tin_files::TIN_HELD),
("tin_files::TIN_PROBE", tin_files::TIN_PROBE),
@@ -10768,6 +10765,69 @@ mod tests {
);
}
/// No tool takes a path on the server's computer (plan 4.7, D-28):
/// files come by upload or inline. The three outputs with full TINs
/// keep theirs until the viewer takes them over (P2 slice 6).
#[test]
fn no_tool_takes_a_path() {
const UNTIL_THE_VIEWER: &[(&str, &str)] = &[
("books.export_cpa_package", "save_to"),
("books.vendor_1099", "save_csv_to"),
("books.get_w9", "save_to"),
];
fn names(v: &serde_json::Value, out: &mut Vec<String>) {
match v {
serde_json::Value::Object(map) => {
if let Some(serde_json::Value::Object(props)) = map.get("properties") {
out.extend(props.keys().cloned());
}
map.values().for_each(|v| names(v, out));
}
serde_json::Value::Array(items) => items.iter().for_each(|v| names(v, out)),
_ => {}
}
}
let mut found = Vec::new();
for t in BooksServer::tool_router().list_all() {
let mut props = Vec::new();
names(
&serde_json::Value::Object((*t.input_schema).clone()),
&mut props,
);
for p in props {
let lower = p.to_ascii_lowercase();
// Names that hold a place to read or write: whole words,
// so `link_to`, `*_file_name`, `tin_on_file` and `date_to`
// pass.
let words: Vec<&str> = lower.split('_').collect();
let pathlike = lower.starts_with("save")
|| lower.contains("path")
|| [
"dir",
"folder",
"root",
"dest",
"destination",
"output",
"location",
"local",
]
.iter()
.any(|w| words.contains(w))
|| (words.last() == Some(&"file") && !lower.ends_with("on_file"))
|| lower == "to"
|| lower == "from"
|| lower == "source";
if pathlike && !UNTIL_THE_VIEWER.contains(&(&*t.name, p.as_str())) {
found.push(format!("{} {p}", t.name));
}
}
}
found.sort();
found.dedup();
assert!(found.is_empty(), "path-like parameters: {found:?}");
}
#[test]
fn tool_text_lint() {
const NEVER: &[&str] = &[
@@ -10793,6 +10853,13 @@ mod tests {
"read the file",
"open the file",
"cat ",
// Gone in P2 slice 5: files come by upload or inline.
"import_local_file",
"w9_file_path",
"import folder",
"import root",
"put_url",
"human_url",
"fl-0001",
];
// Whole words only: "digit" and "committee" are fine.
@@ -10830,6 +10897,45 @@ mod tests {
));
}
}
// Every tool the text names is one there is.
let tools: std::collections::HashSet<String> = BooksServer::tool_router()
.list_all()
.into_iter()
.map(|t| t.name.to_string())
.collect();
let mut unknown: Vec<String> = Vec::new();
// (Settings keys, `books.cash_accounts`, are named like tools.)
for (what, text) in texts
.iter()
.filter(|(w, _)| !w.contains("settings literal"))
{
for (i, _) in text.match_indices("books.") {
let name: String = text[i..]
.chars()
.take_while(|c| {
c.is_ascii_lowercase() || c.is_ascii_digit() || *c == '_' || *c == '.'
})
.collect();
let name = name.trim_end_matches('.');
// "the books." ends a sentence.
if name == "books" {
continue;
}
let before = text[..i].chars().next_back();
if before.is_some_and(|c| c.is_alphanumeric() || c == '_' || c == '/') {
continue;
}
if !tools.contains(name) {
unknown.push(format!("{what}: {name}"));
}
}
}
unknown.sort();
unknown.dedup();
assert!(
unknown.is_empty(),
"text names tools there aren't: {unknown:?}"
);
let words = |text: &str| {
text.split(|c: char| !c.is_alphanumeric() && c != '_')
.map(str::to_string)
crates/ledger-mcp/src/tools/files.rs
+266
View File
@@ -0,0 +1,266 @@
//! Files in (plan 4.7, P2 slice 5, D-28): `books.begin_upload`, and the
//! bytes a tool is given, as an upload or inline.
//!
//! ```text
//! GOAL: an agent files a document of any size up to the limit without
//! its bytes passing through its context, and a small one inline,
//! both exactly as the agent declared them.
//! SOURCE: plan 4.7 (begin_upload; "content_base64 (≤256 KiB + required
//! sha256)"; guard base64_corrupt), 2.1 ("256 KiB with a mandatory
//! matching sha256"), D-28 (the agent does it all).
//! INVARIANTS:
//! 1. A tool's bytes are an upload made by the same token, or inline
//! base64 of at most INLINE_MAX bytes whose SHA-256 is the one given;
//! never both, never neither. An upload is used up only by the call
//! that files it (TakenFile::filed); a call that fails leaves it for
//! the next.
//! 2. begin_upload makes nothing for a receipt or document whose bytes
//! are stored already: it names the attachment (not withdrawn) that
//! holds them.
//! 3. Uploads exist only for books served from the data root (the
//! server's socket carries the bytes); over stdio, only inline.
//! 4. An upload is filed only by a tool of the purpose it was begun for
//! (a receipt or document, a CSV import, a W-9); another tool's
//! refusal leaves it for the right one.
//! 5. Inline text longer than INLINE_MAX bytes' base64 is refused
//! before it is decoded.
//! ```
use super::*;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
/// The most bytes given inline (plan 2.1: 256 KiB).
pub(crate) const INLINE_MAX: usize = 256 * 1024;
/// Why an upload is refused on a connection that has none (INVARIANT 3).
const STDIO_ONLY_INLINE: &str = "this connection takes files inline only (content_base64 with its sha256, at most 256 KiB): a larger one needs these books served by `wanderbooks mcp`; tell the human";
/// What the bytes of a file a tool files came as (INVARIANT 1), and the
/// upload purposes the tool takes (INVARIANT 4).
pub(crate) struct Given<'a> {
pub upload_id: Option<&'a str>,
pub content_base64: Option<&'a str>,
pub sha256: Option<&'a str>,
pub purposes: &'a [&'a str],
}
/// What attach_receipt and file_document take.
pub(crate) const FILED: &[&str] = &["receipt", "document"];
/// What the CSV imports take.
pub(crate) const IMPORTED: &[&str] = &["import"];
/// What set_w9 takes.
pub(crate) const W9: &[&str] = &["w9"];
/// The bytes, and the name and type an upload declared.
pub(crate) struct TakenFile {
pub bytes: Vec<u8>,
pub file_name: Option<String>,
pub mime: Option<String>,
/// The upload's hold, if it came as one (INVARIANT 1).
pub claim: Option<ledger_store::uploads::Claim>,
}
/// ` --token NAME` for a command given to the agent, when its token isn't
/// the one the commands use by default: the command runs as the caller.
pub(crate) fn token_flag() -> String {
CALLER.with(|c| match c.borrow().as_ref() {
Some((_, name)) if name != ledger_store::auth::DEFAULT_NAME => {
format!(" --token {name}")
}
_ => String::new(),
})
}
/// The bytes are in the books: an upload they came as is used up
/// (INVARIANT 1). Dropped instead, it is given back.
pub(crate) fn filed(claim: Option<ledger_store::uploads::Claim>) {
if let Some(c) = claim {
c.spend();
}
}
#[derive(Debug, Deserialize, JsonSchema)]
pub struct BeginUploadParams {
/// The file's SHA-256, hex (`sha256sum FILE`).
pub sha256: String,
/// Its size in bytes (at most 25 MiB).
pub size: u64,
/// Its MIME type, e.g. "application/pdf".
pub mime_type: String,
/// Its name, e.g. "receipt.pdf".
pub file_name: String,
/// What it is for: receipt, document, import (a bank or chart CSV), or
/// w9 (books.set_w9).
pub purpose: String,
}
#[derive(Debug, Serialize, JsonSchema)]
pub struct BeginUploadOut {
/// The upload's id: name it in the tool that files it.
#[serde(skip_serializing_if = "Option::is_none")]
pub upload_id: Option<String>,
/// The command that sends the file (run it with the file's path).
#[serde(skip_serializing_if = "Option::is_none")]
pub command: Option<String>,
/// When the upload lapses (UTC).
#[serde(skip_serializing_if = "Option::is_none")]
pub expires_at: Option<String>,
/// The most bytes taken.
pub max_bytes: u64,
/// These bytes are stored already, in this attachment: link it
/// (books.link_attachment) instead of filing a copy.
#[serde(skip_serializing_if = "Option::is_none")]
pub existing_attachment_id: Option<u64>,
}
impl BooksServer {
/// The token of the call running on this thread (its id), if any.
fn caller_token(&self) -> Option<String> {
CALLER.with(|c| c.borrow().as_ref().map(|(id, _)| id.clone()))
}
/// The bytes a tool was given (INVARIANT 1).
pub(crate) fn take_file(&self, given: Given<'_>) -> Result<TakenFile, String> {
match (given.upload_id, given.content_base64) {
(Some(_), Some(_)) => Err("give upload_id or content_base64, not both".into()),
(None, None) => Err(
"the file's bytes are missing: upload it (books.begin_upload), or give content_base64 with its sha256 (at most 256 KiB)"
.into(),
),
(Some(id), None) => {
// INVARIANT 3.
let (Some(uploads), Some(token)) = (&self.uploads, self.caller_token()) else {
return Err(STDIO_ONLY_INLINE.into());
};
let got = uploads
.take(id.trim(), &token, chrono::Utc::now().timestamp())
.map_err(|e| e.to_string())?;
// INVARIANT 4: dropped, the claim gives it back.
if !given.purposes.contains(&got.grant.purpose.as_str()) {
return Err(format!(
"that upload was begun for {}: this tool takes one begun for {}",
got.grant.purpose,
given.purposes.join(" or ")
));
}
Ok(TakenFile {
bytes: got.bytes,
file_name: Some(got.grant.file_name),
mime: Some(got.grant.mime),
claim: Some(got.claim),
})
}
(None, Some(b64)) => {
let Some(sha) = given.sha256 else {
return Err("content_base64 comes with its sha256 (the file's SHA-256, hex), so a byte lost on the way is caught".into());
};
// INVARIANT 5.
if b64.len() > INLINE_MAX.div_ceil(3) * 4 {
return Err(format!(
"inline is at most {INLINE_MAX} bytes: upload it with books.begin_upload"
));
}
let bytes = base64::Engine::decode(&base64::engine::general_purpose::STANDARD, b64)
.map_err(|_| "content_base64 is not valid base64 (guard base64_corrupt): send the file with books.begin_upload".to_string())?;
if bytes.len() > INLINE_MAX {
return Err(format!(
"the file is {} bytes; inline is at most {INLINE_MAX}: upload it with books.begin_upload",
bytes.len()
));
}
if ledger_store::journal::sha256_hex(&bytes) != sha.trim().to_ascii_lowercase() {
return Err("content_base64 doesn't have the sha256 given: some of it was lost on the way (guard base64_corrupt). Send the file with books.begin_upload instead".into());
}
Ok(TakenFile {
bytes,
file_name: None,
mime: None,
claim: None,
})
}
}
}
}
#[tool_router(router = files_router, vis = "pub(super)")]
impl BooksServer {
#[tool(
name = "books.begin_upload",
title = "Begin an upload",
description = "Send a file of up to 25 MiB (books served by `wanderbooks mcp`; elsewhere only inline) without its bytes passing through you: give its sha256 (sha256sum), size, type, name and purpose; run the command returned with the file's path; then name the upload_id in books.attach_receipt, books.file_document, books.set_w9, books.import_bank_csv or books.import_chart_csv. Purpose: receipt or document (those two tools), import (the CSV imports), w9. If a receipt's or document's bytes are stored already, existing_attachment_id names them: link that instead. An upload lasts 15 minutes and is used once.",
annotations(
read_only_hint = false,
destructive_hint = false,
open_world_hint = false
)
)]
async fn begin_upload(
&self,
params: Parameters<BeginUploadParams>,
) -> Result<Json<BeginUploadOut>, String> {
let p = params.0;
let max = ledger_store::uploads::MAX_UPLOAD;
let sha = p.sha256.trim().to_ascii_lowercase();
if !ledger_store::uploads::PURPOSES.contains(&p.purpose.as_str()) {
return Err("purpose is receipt, document, import (a bank or chart CSV) or w9".into());
}
// INVARIANT 2: an attachment that can be linked, for a file that
// would be one (an import or a W-9 needs its bytes).
let store = self.lock()?;
let existing = if FILED.contains(&p.purpose.as_str()) {
store
.list_attachments(None)
.map_err(|e| e.to_string())?
.into_iter()
.find(|a| a.blob_sha == sha && a.withdrawn_at.is_none())
.map(|a| a.id)
} else {
None
};
let book = store
.data_dir()
.file_name()
.map(|n| n.to_string_lossy().to_string())
.unwrap_or_default();
drop(store);
if let Some(id) = existing {
return Ok(Json(BeginUploadOut {
upload_id: None,
command: None,
expires_at: None,
max_bytes: max,
existing_attachment_id: Some(id),
}));
}
let (Some(uploads), Some(token)) = (&self.uploads, self.caller_token()) else {
return Err(STDIO_ONLY_INLINE.into());
};
let now = chrono::Utc::now();
let grant = uploads
.grant(
&token,
&sha,
p.size,
&p.mime_type,
&p.file_name,
&p.purpose,
now.timestamp(),
)
.map_err(|e| e.to_string())?;
let expires = chrono::DateTime::from_timestamp(grant.expires, 0)
.map(|t| t.to_rfc3339_opts(chrono::SecondsFormat::Secs, true))
.unwrap_or_default();
Ok(Json(BeginUploadOut {
command: Some(format!(
"wanderbooks upload --book {book}{} {} -- <the file's path>",
token_flag(),
grant.id
)),
upload_id: Some(grant.id),
expires_at: Some(expires),
max_bytes: max,
existing_attachment_id: None,
}))
}
}
crates/ledger-mcp/src/types.rs
+80 -65
View File
@@ -369,7 +369,7 @@ pub struct Vendor1099Params {
pub include_tins: bool,
/// Also write the worksheet as CSV to this local path (TINs included
/// only with include_tins). Never inside the books dir, and with TINs
/// never in the server's import folders; an existing file is refused.
/// an existing file is refused.
pub save_csv_to: Option<String>,
}
@@ -868,8 +868,12 @@ pub struct CashFlowOut {
#[derive(Debug, Deserialize, JsonSchema)]
pub struct ImportBankCsvParams {
/// The CSV text (QBO bank upload layout: Date,Description,Amount, or
/// Date,Description, money out, money in).
pub csv_text: String,
/// Date,Description, money out, money in)…
#[serde(default)]
pub csv_text: Option<String>,
/// …or an upload of it made with books.begin_upload (purpose import).
#[serde(default)]
pub upload_id: Option<String>,
/// Account code for the bank account (the offset side of each entry).
pub bank_account_code: String,
/// Account code for the default offset (e.g. an expense account).
@@ -923,8 +927,12 @@ pub struct ImportBankCsvOut {
/// Parameters for `books.import_chart_csv`.
#[derive(Debug, Deserialize, JsonSchema)]
pub struct ImportChartCsvParams {
/// The CSV text: code,name,type,parent_code with a header row.
pub csv_text: String,
/// The CSV text: code,name,type,parent_code with a header row…
#[serde(default)]
pub csv_text: Option<String>,
/// …or an upload of it made with books.begin_upload (purpose import).
#[serde(default)]
pub upload_id: Option<String>,
}
/// Result of `books.import_chart_csv`.
@@ -936,41 +944,6 @@ pub struct ImportChartCsvOut {
pub count: usize,
}
/// Parameters for `books.import_local_file`.
#[derive(Debug, Deserialize, JsonSchema)]
pub struct ImportLocalFileParams {
/// Absolute path to a regular file in the server's import folders (by
/// default your downloads folder).
pub path: String,
/// "receipt" or "document".
pub category: String,
/// Entry id to attach to (receipts only). Omit to file unlinked.
#[serde(default)]
pub entry_id: Option<u64>,
/// Document category for documents: formation, legal, tax, accounting, other.
#[serde(default)]
pub document_category: Option<String>,
/// Override the file name (defaults to the path's file name).
#[serde(default)]
pub file_name: Option<String>,
/// Optional description.
#[serde(default)]
pub description: Option<String>,
}
/// Result of `books.import_local_file`.
#[derive(Debug, Serialize, JsonSchema)]
pub struct ImportLocalFileOut {
/// Attachment id.
pub attachment_id: u64,
/// SHA-256 hex of the content.
pub sha256: String,
/// True if the content was already stored (deduplicated).
pub deduplicated: bool,
/// File name used.
pub file_name: String,
}
/// Parameters for `books.balance_sheet`.
#[derive(Debug, Deserialize, JsonSchema)]
pub struct BalanceSheetParams {
@@ -1270,12 +1243,23 @@ pub struct AttachReceiptParams {
/// Entry id to attach to. Omit to file unlinked, then link later with
/// `books.link_receipt`.
pub entry_id: Option<u64>,
/// Original file name, e.g. "receipt.png".
pub file_name: String,
/// MIME type, e.g. "image/png".
pub mime_type: String,
/// File bytes, base64-encoded. Max 25 MB.
pub content_base64: String,
/// Original file name, e.g. "receipt.png" (an upload's own, if not
/// given).
#[serde(default)]
pub file_name: Option<String>,
/// MIME type, e.g. "image/png" (an upload's own, if not given).
#[serde(default)]
pub mime_type: Option<String>,
/// The file's bytes: an upload made with books.begin_upload (its id)…
#[serde(default)]
pub upload_id: Option<String>,
/// …or, for a file of at most 256 KiB, its bytes base64-encoded, with
/// sha256.
#[serde(default)]
pub content_base64: Option<String>,
/// The SHA-256 of the bytes in content_base64, hex.
#[serde(default)]
pub sha256: Option<String>,
/// Optional description.
pub description: Option<String>,
/// More subjects to link it to in the same call (each journaled as its
@@ -1304,12 +1288,22 @@ pub struct FileDocumentParams {
/// One of: formation, legal, tax, accounting, statement, contract,
/// photo, warranty, import_source, other.
pub category: String,
/// Original file name.
pub file_name: String,
/// MIME type.
pub mime_type: String,
/// File bytes, base64-encoded. Max 25 MB.
pub content_base64: String,
/// Original file name (an upload's own, if not given).
#[serde(default)]
pub file_name: Option<String>,
/// MIME type (an upload's own, if not given).
#[serde(default)]
pub mime_type: Option<String>,
/// The file's bytes: an upload made with books.begin_upload (its id)…
#[serde(default)]
pub upload_id: Option<String>,
/// …or, for a file of at most 256 KiB, its bytes base64-encoded, with
/// sha256.
#[serde(default)]
pub content_base64: Option<String>,
/// The SHA-256 of the bytes in content_base64, hex.
#[serde(default)]
pub sha256: Option<String>,
/// Optional description.
pub description: Option<String>,
/// Subjects to link it to in the same call (each journaled as its own
@@ -1421,10 +1415,16 @@ pub struct ListReceiptsOut {
pub struct GetReceiptParams {
/// Attachment id.
pub attachment_id: u64,
/// What to return: "meta" (default: what it is and what it's linked
/// to), or "bytes" (also its bytes, base64, for a file of at most
/// 256 KiB; a larger one comes with `wanderbooks download`).
#[serde(default)]
pub view: Option<String>,
}
/// Result of `books.get_receipt`: metadata plus base64 bytes. The bytes are
/// hash-verified on read.
/// Result of `books.get_receipt`: what it is and what it's linked to, and
/// with view `bytes`, its bytes (hash-verified on read) or the command that
/// saves a copy.
#[derive(Debug, Serialize, JsonSchema)]
pub struct GetReceiptOut {
/// Attachment id.
@@ -1445,8 +1445,14 @@ pub struct GetReceiptOut {
pub description: Option<String>,
/// Everything it is linked to (the entry it is filed on first).
pub links: Vec<SubjectOut>,
/// File bytes, base64-encoded.
pub content_base64: String,
/// Its bytes, base64-encoded: with view `bytes`, for a file of at most
/// 256 KiB.
#[serde(skip_serializing_if = "Option::is_none")]
pub content_base64: Option<String>,
/// With view `bytes`, for a file too large to come back here: the
/// command that saves a copy in a folder you name.
#[serde(skip_serializing_if = "Option::is_none")]
pub download: Option<String>,
}
// ---------------------------------------------------------------------------
@@ -2605,8 +2611,7 @@ pub struct ExportCpaPackageParams {
pub include_tins: bool,
/// Where to write the package: a new `.zip` file, or an existing
/// folder (the zip goes in it as `cpa-package-<as_of>.zip`). Written
/// readable by its owner only. With TINs, never inside the books dir
/// or the server's import folders.
/// readable by its owner only. With TINs, never inside the books dir.
pub save_to: Option<String>,
}
@@ -2727,11 +2732,22 @@ pub struct SetW9Params {
pub tin: Option<SecretTin>,
/// The TIN's type: ssn, ein, or itin. Required with tin.
pub tin_type: Option<String>,
/// A local PDF, PNG, or JPEG of the signed W-9, in the server's import
/// folders (by default your downloads folder): stored only in the
/// vault (it shows the full TIN), never with the receipts and
/// documents. Read it back with `books.get_w9`.
pub w9_file_path: Option<String>,
/// The signed W-9 itself (a PDF, PNG, or JPEG): an upload made with
/// books.begin_upload. Stored only in the vault (it shows the full
/// TIN), never with the receipts and documents.
#[serde(default)]
pub w9_upload_id: Option<String>,
/// …or, for one of at most 256 KiB, its bytes base64-encoded, with
/// w9_sha256.
#[serde(default)]
pub w9_content_base64: Option<String>,
/// The SHA-256 of the bytes in w9_content_base64, hex.
#[serde(default)]
pub w9_sha256: Option<String>,
/// The W-9 file's name (default: the upload's, else w9.pdf, w9.png
/// or w9.jpg by its type).
#[serde(default)]
pub w9_file_name: Option<String>,
}
/// A full TIN as a tool takes it: text (or a JSON number) whose value is
@@ -2807,8 +2823,7 @@ pub struct GetW9Params {
/// The party: id or name.
pub party: String,
/// Where to write the W-9 file: a new local file outside the books
/// and outside the server's import folders (the file shows the full
/// TIN). An existing file is refused.
/// (the file shows the full TIN). An existing file is refused.
pub save_to: String,
}