From b26c099a6882c81abc27047df2b45c847678617c Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Mon, 19 Oct 2026 13:44:50 -0600 Subject: [PATCH] P2 slice 5: files in and out Files reach the books through the server: uploads by hash and size, then filed; exports (the CPA package, the 1099 worksheet, W-9s) handed over as downloads from the server, never read off disk by the agent. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01T3gXMVt7XJPi6ens1or9Dj --- Cargo.lock | 4 + crates/ledger-cli/Cargo.toml | 2 + crates/ledger-cli/src/main.rs | 53 +- crates/ledger-cli/tests/local_server.rs | 832 +++++++++- crates/ledger-cli/tests/one_executable.rs | 6 - crates/ledger-cli/tests/open_while_held.rs | 295 +++- crates/ledger-host/Cargo.toml | 2 + crates/ledger-host/src/bridge.rs | 4 +- crates/ledger-host/src/commands.rs | 666 ++++++++ crates/ledger-host/src/tokens.rs | 2 +- crates/ledger-mcp/Cargo.toml | 8 +- crates/ledger-mcp/handbook/1099.md | 2 +- crates/ledger-mcp/handbook/cpa-handoff.md | 2 +- crates/ledger-mcp/handbook/files-in-out.md | 10 +- .../ledger-mcp/handbook/parties-and-tins.md | 2 +- crates/ledger-mcp/src/lib.rs | 61 +- crates/ledger-mcp/src/local.rs | 517 +++++- crates/ledger-mcp/src/local_files.rs | 1430 ----------------- crates/ledger-mcp/src/paths.rs | 132 ++ crates/ledger-mcp/src/scopes.rs | 2 +- crates/ledger-mcp/src/server.rs | 642 +++++--- crates/ledger-mcp/src/tools/files.rs | 266 +++ crates/ledger-mcp/src/types.rs | 145 +- crates/ledger-mcp/tests/hostile_args.rs | 58 +- crates/ledger-mcp/tests/http_concurrency.rs | 3 +- crates/ledger-mcp/tests/mcp_stdio.rs | 321 +--- crates/ledger-mcp/tests/oracle_mcp.rs | 321 +++- crates/ledger-mcp/tests/spec_assets.rs | 4 +- crates/ledger-mcp/tests/spec_attachments.rs | 21 +- crates/ledger-mcp/tests/spec_mcp.rs | 77 +- crates/ledger-mcp/tests/spec_vault.rs | 179 ++- crates/ledger-mcp/tests/startup.rs | 202 +-- crates/ledger-mcp/tests/tin_files.rs | 109 +- crates/ledger-store/src/auth.rs | 4 + crates/ledger-store/src/lib.rs | 1 + crates/ledger-store/src/uploads.rs | 1045 ++++++++++++ docs/mcp-interface.md | 301 ++-- docs/parity.md | 5 +- docs/parties-and-attachments.md | 5 +- docs/receipt-store.md | 9 +- docs/spec/rules.md | 13 +- docs/testing.md | 36 +- docs/verification.md | 3 +- plan/served.md | 12 +- 44 files changed, 5075 insertions(+), 2739 deletions(-) delete mode 100644 crates/ledger-mcp/src/local_files.rs create mode 100644 crates/ledger-mcp/src/paths.rs create mode 100644 crates/ledger-mcp/src/tools/files.rs create mode 100644 crates/ledger-store/src/uploads.rs diff --git a/Cargo.lock b/Cargo.lock index 76912f7..b1a793c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3824,6 +3824,7 @@ checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" name = "ledger-cli" version = "0.1.0" dependencies = [ + "base64 0.22.1", "chrono", "ledger-config", "ledger-core", @@ -3836,6 +3837,7 @@ dependencies = [ "reqwest", "rmcp", "serde_json", + "sha2 0.10.9", "tempfile", "tokio", "toml 1.1.8+spec-1.1.0", @@ -3908,6 +3910,7 @@ dependencies = [ "rpassword", "serde", "serde_json", + "sha2 0.10.9", "tempfile", "tokio", "tokio-util", @@ -3960,6 +3963,7 @@ dependencies = [ "schemars", "serde", "serde_json", + "sha2 0.10.9", "tempfile", "thiserror 2.0.21", "tokio", diff --git a/crates/ledger-cli/Cargo.toml b/crates/ledger-cli/Cargo.toml index 563b94c..b2473de 100644 --- a/crates/ledger-cli/Cargo.toml +++ b/crates/ledger-cli/Cargo.toml @@ -23,6 +23,7 @@ ledger-store = { path = "../ledger-store" } ledger-gui = { path = "../ledger-gui", optional = true } [dev-dependencies] +base64 = "0.22" chrono = "0.4" libc = "0.2" reqwest = "0.13" @@ -32,6 +33,7 @@ ledger-config = { path = "../ledger-config" } ledger-core = { path = "../ledger-core" } ledger-vault = { path = "../ledger-vault" } serde_json = "1" +sha2 = "0.10" toml = "1" tempfile = "3" diff --git a/crates/ledger-cli/src/main.rs b/crates/ledger-cli/src/main.rs index 2220981..6d8cf84 100644 --- a/crates/ledger-cli/src/main.rs +++ b/crates/ledger-cli/src/main.rs @@ -400,6 +400,8 @@ fn print_usage() { ); eprintln!(" wanderbooks mcp --book ID [--token NAME]"); eprintln!(" wanderbooks passphrase"); + eprintln!(" wanderbooks upload --book ID [--token NAME] UPLOAD [--] FILE"); + eprintln!(" wanderbooks download --book ID [--token NAME] ATTACHMENT --to FOLDER"); eprintln!(" wanderbooks serve | close | doctor"); eprintln!(); eprintln!(" new make a books directory and start the agent in it (alias: init)"); @@ -410,6 +412,8 @@ fn print_usage() { eprintln!(" token make, list or revoke an agent's token for books (`token --help`)"); eprintln!(" mcp connect an agent to books, over stdio (what `claude mcp add` runs)"); eprintln!(" passphrase set or change the owner passphrase (typed at a terminal)"); + eprintln!(" upload send a file for an upload an agent began (books.begin_upload)"); + eprintln!(" download save a copy of a stored file (one too large to come back inline)"); eprintln!(" serve hold the books open until Ctrl-C"); eprintln!(" close close the books now: everything holding them lets go"); eprintln!(" doctor whether the books are open, what holds them, what went wrong"); @@ -994,6 +998,34 @@ fn run_local(mode: &str, args: Vec) -> i32 { } } +/// `upload` and `download`: the agent's file commands, whose words may +/// be paths that aren't text. +#[cfg(unix)] +fn run_files(mode: &str, args: Vec) -> i32 { + let prog = format!("wanderbooks {mode}"); + let result = match mode { + "upload" => ledger_host::commands::upload(&prog, args), + _ => ledger_host::commands::download(&prog, args), + }; + match result { + Ok(()) => 0, + Err(e) => match e.downcast_ref::() { + Some(exit) => exit.0, + None => { + eprintln!("error: {e}"); + 1 + } + }, + } +} + +/// The file commands, where local sockets aren't built yet. +#[cfg(not(unix))] +fn run_files(_mode: &str, _args: Vec) -> i32 { + eprintln!("error: opening the books this way comes to this system with P11"); + 2 +} + /// The modes that hold the books, where local sockets aren't built yet. #[cfg(not(unix))] fn run_local(_mode: &str, _args: Vec) -> i32 { @@ -1009,8 +1041,23 @@ fn run_server(_args: Vec) -> i32 { } fn main() { + // A file's path may be any name the system takes: the file commands + // get the words as given. Everything else is text. + let given: Vec = env::args_os().skip(1).collect(); + if let Some(mode @ ("upload" | "download")) = given.first().and_then(|m| m.to_str()) { + let mode = mode.to_string(); + std::process::exit(run_files(&mode, given[1..].to_vec())); + } + let Ok(words) = given + .into_iter() + .map(std::ffi::OsString::into_string) + .collect::, _>>() + else { + eprintln!("error: an argument isn't UTF-8 text"); + std::process::exit(2); + }; // The modes run in this process and take their own flags. - let mut all = env::args().skip(1); + let mut all = words.clone().into_iter(); match all.next().as_deref() { Some("gui") => std::process::exit(run_gui(all.collect())), Some("dev-stdio") => std::process::exit(run_dev_stdio(all.collect())), @@ -1030,7 +1077,7 @@ fn main() { let mut start = true; let mut agent = Agent::Claude; let mut cmd: Option = None; - for arg in env::args().skip(1) { + for arg in words { match arg.as_str() { "--help" | "-h" => { print_usage(); @@ -1055,7 +1102,7 @@ fn main() { } if let Some( mode @ ("gui" | "dev-stdio" | "setup" | "server" | "host" | "helper" | "mcp" | "serve" - | "close" | "books" | "doctor" | "token" | "passphrase"), + | "close" | "books" | "doctor" | "token" | "passphrase" | "upload" | "download"), ) = cmd.as_deref() { eprintln!("`{mode}` must come first: wanderbooks {mode} [its own flags]"); diff --git a/crates/ledger-cli/tests/local_server.rs b/crates/ledger-cli/tests/local_server.rs index 99e90c8..c536926 100644 --- a/crates/ledger-cli/tests/local_server.rs +++ b/crates/ledger-cli/tests/local_server.rs @@ -265,7 +265,8 @@ async fn post_one(c: &Client, n: u64) -> Result { c, "books.attach_receipt", json!({"entry_id": id, "file_name": format!("r{n}.txt"), - "mime_type": "text/plain", "content_base64": "cmVjZWlwdA=="}), + "mime_type": "text/plain", "content_base64": "cmVjZWlwdA==", + "sha256": "6f32860910ca0fb2a20c7fda143666b09dbf8db5238195c90a586fb542ff0cad"}), ) .await?; call( @@ -747,6 +748,42 @@ async fn raw_session(http: &reqwest::Client, book: &str) -> String { session } +/// Begin an upload on `session` (books.begin_upload) and send `bytes` for +/// it over the upload route: its id. +async fn raw_upload(http: &reqwest::Client, session: &str, bytes: &[u8], name: &str) -> String { + let begin = json!({"jsonrpc": "2.0", "id": 9, "method": "tools/call", "params": { + "name": "books.begin_upload", "arguments": { + "sha256": ledger_store::journal::sha256_hex(bytes), "size": bytes.len(), + "mime_type": "application/pdf", "file_name": name, "purpose": "document"}}}); + let r = http + .post("http://localhost/b/acme/mcp") + .header("content-type", "application/json") + .header("accept", "application/json, text/event-stream") + .header("mcp-session-id", session) + .header("mcp-protocol-version", "2025-06-18") + .body(begin.to_string()) + .send() + .await + .unwrap() + .text() + .await + .unwrap(); + let id = r + .split("\"upload_id\":\"") + .nth(1) + .and_then(|rest| rest.split('"').next()) + .unwrap_or_else(|| panic!("no upload: {r}")) + .to_string(); + let put = http + .put(format!("http://localhost/b/acme/u/{id}")) + .body(bytes.to_vec()) + .send() + .await + .unwrap(); + assert_eq!(put.status().as_u16(), 200, "{:?}", put.text().await); + id +} + /// Wait for a child to exit, at most `secs`. fn wait_exit(child: &mut Child, secs: u64) -> Option { let deadline = Instant::now() + Duration::from_secs(secs); @@ -768,14 +805,14 @@ async fn a_call_whose_client_hung_up_finishes_before_the_close() { home.create("acme", "USD"); let http = mcp_http_for(&home, "acme"); let session = raw_session(&http, "acme").await; - // A large document: its filing takes long enough to close mid-way. + // A large document, sent as an upload: its filing takes long enough to + // close mid-way. let mut bytes = b"%PDF-1.4\n".to_vec(); bytes.extend((0..20_000_000u32).map(|i| (i % 251) as u8)); - use base64_engine::encode; + let upload = raw_upload(&http, &session, &bytes, "big.pdf").await; let call = json!({"jsonrpc": "2.0", "id": 2, "method": "tools/call", "params": { "name": "books.file_document", "arguments": { - "category": "formation", "file_name": "big.pdf", - "mime_type": "application/pdf", "content_base64": encode(&bytes)}}}); + "category": "formation", "upload_id": upload}}}); let response = http .post("http://localhost/b/acme/mcp") .header("content-type", "application/json") @@ -810,27 +847,6 @@ async fn a_call_whose_client_hung_up_finishes_before_the_close() { assert_eq!(snap.attachment_bytes(filed[0].id).unwrap(), bytes); } -/// Base64 for the tests (standard alphabet, padded). -mod base64_engine { - pub fn encode(bytes: &[u8]) -> String { - const T: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - let mut out = String::with_capacity(bytes.len() * 4 / 3 + 4); - for c in bytes.chunks(3) { - let n = (u32::from(c[0]) << 16) - | (u32::from(*c.get(1).unwrap_or(&0)) << 8) - | u32::from(*c.get(2).unwrap_or(&0)); - for (i, shift) in [18, 12, 6, 0].iter().enumerate() { - out.push(if i <= c.len() { - T[((n >> shift) & 63) as usize] as char - } else { - '=' - }); - } - } - out - } -} - /// Send the start of a request that never finishes. fn stall(home: &Home) -> std::os::unix::net::UnixStream { use std::io::Write; @@ -1767,3 +1783,767 @@ async fn the_vault_and_the_approval_controls_need_their_scopes() { drop((w, p)); assert!(server.close().success()); } + +// P2 slice 5: files in and out (ledger-mcp tools/files.rs, local.rs's +// upload and download routes; ledger-store uploads.rs). + +fn b64(bytes: &[u8]) -> String { + base64::Engine::encode(&base64::engine::general_purpose::STANDARD, bytes) +} + +/// Begin an upload on `c` for `bytes`: its id. +async fn begin(c: &Client, bytes: &[u8], name: &str, purpose: &str) -> String { + let r = call( + c, + "books.begin_upload", + json!({"sha256": ledger_store::journal::sha256_hex(bytes), "size": bytes.len(), + "mime_type": "application/pdf", "file_name": name, "purpose": purpose}), + ) + .await + .unwrap(); + r["upload_id"].as_str().unwrap().to_string() +} + +/// PUT `bytes` for upload `id` with `token`: the status, the refusal +/// code and the body. +async fn put(home: &Home, token: &str, id: &str, bytes: &[u8]) -> (u16, Option, String) { + let r = mcp_http(home) + .put(format!("http://localhost/b/acme/u/{id}")) + .bearer_auth(token) + .body(bytes.to_vec()) + .send() + .await + .unwrap(); + let status = r.status().as_u16(); + let code = r + .headers() + .get(ledger_mcp::local::REFUSED_HEADER) + .map(|v| v.to_str().unwrap().to_string()); + (status, code, r.text().await.unwrap()) +} + +/// A receipt-like PDF of `size` bytes, different for each `tag`. +fn pdf(tag: &str, size: usize) -> Vec { + let mut b = format!("%PDF-1.4 {tag} ").into_bytes(); + let mut n: u32 = 0; + while b.len() < size { + b.extend_from_slice(&n.to_le_bytes()); + n = n.wrapping_mul(1_103_515_245).wrapping_add(12_345); + } + b.truncate(size); + b +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_upload_is_filed_once_by_its_own_token_exactly_as_declared() { + let home = Home::new("upload"); + let server = Server::start(&home); + home.create("acme", "USD"); + let c = connect(&home, "acme").await.unwrap(); + let token = home.token("acme"); + let bytes = pdf("one", 300 * 1024); + // The command it gives runs with the agent's own path; it names no + // path of the server's. + let r = call( + &c, + "books.begin_upload", + json!({"sha256": ledger_store::journal::sha256_hex(&bytes), "size": bytes.len(), + "mime_type": "application/pdf", "file_name": "big receipt.pdf", + "purpose": "document"}), + ) + .await + .unwrap(); + home.assert_hidden(&r.to_string()); + let id = r["upload_id"].as_str().unwrap().to_string(); + assert_eq!( + r["command"], + json!(format!( + "wanderbooks upload --book acme --token test {id} -- " + )) + ); + assert_eq!(r["max_bytes"], json!(25 * 1024 * 1024)); + assert!(r["expires_at"].as_str().unwrap().ends_with('Z')); + // Not sent yet: nothing to file. + let e = call( + &c, + "books.file_document", + json!({"category": "other", "upload_id": id}), + ) + .await + .unwrap_err(); + assert!(e.contains("haven't been sent"), "{e}"); + // Anything but the bytes declared is refused and kept nowhere. + let mut short = bytes.clone(); + short.pop(); + let mut changed = bytes.clone(); + changed[100] ^= 1; + for wrong in [&short, &changed] { + let (status, code, _) = put(&home, &token, &id, wrong).await; + assert_eq!((status, code.as_deref()), (400, Some("not_as_declared"))); + } + // Another token's upload takes nothing from this one. + let other = home.token_with("acme", "other", &["read", "write"]); + let (status, code, _) = put(&home, &other, &id, &bytes).await; + assert_eq!((status, code.as_deref()), (403, Some("not_yours"))); + let (status, code, _) = put(&home, &token, "up_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", &bytes).await; + assert_eq!((status, code.as_deref()), (404, Some("no_such_upload"))); + let (status, _, body) = put(&home, &token, &id, &bytes).await; + assert_eq!(status, 200, "{body}"); + let got: Value = serde_json::from_str(&body).unwrap(); + assert_eq!(got["size"], json!(bytes.len())); + let (status, code, _) = put(&home, &token, &id, &bytes).await; + assert_eq!((status, code.as_deref()), (409, Some("already_sent"))); + // No token: refused at the door, and recorded as an upload too; what + // was sent (25 MiB, more than a socket holds) is read, so the client + // gets the answer. + let r = mcp_http(&home) + .put(format!("http://localhost/b/acme/u/{id}")) + .body(vec![b'x'; 25 * 1024 * 1024]) + .send() + .await + .unwrap(); + assert_eq!(r.status().as_u16(), 401); + // Every PUT is in the access log, refusals too, under its token. + let log = call( + &c, + "books.access_log", + json!({"action": "upload", "limit": 20}), + ) + .await + .unwrap(); + let seen: Vec<(String, String)> = log["records"] + .as_array() + .unwrap() + .iter() + .rev() + .map(|r| { + ( + r["principal"]["label"].as_str().unwrap().to_string(), + r["result"].to_string(), + ) + }) + .collect(); + let outcome = |s: &(String, String)| s.1.clone(); + assert_eq!(seen.len(), 7, "{seen:?}"); + assert!( + outcome(&seen[0]).contains("not_as_declared") + && outcome(&seen[1]).contains("not_as_declared") + ); + assert_eq!(seen[2].0, "other"); + assert!(outcome(&seen[2]).contains("not_yours")); + assert!(outcome(&seen[3]).contains("no_such_upload")); + assert_eq!(outcome(&seen[4]), "\"ok\""); + assert!(outcome(&seen[5]).contains("already_sent")); + let at_door = &log["records"][0]; + assert_eq!(at_door["subject"], json!(id), "{at_door}"); + assert!( + at_door["result"].to_string().contains("refused"), + "{at_door}" + ); + // Only the token that began it files it. + let c2 = connect_with(&home, "acme", Some(other.clone())) + .await + .unwrap(); + let e = call( + &c2, + "books.file_document", + json!({"category": "other", "upload_id": id}), + ) + .await + .unwrap_err(); + assert!(e.contains("another token"), "{e}"); + // Filed once, under the name and type it was declared with. + let filed = call( + &c, + "books.file_document", + json!({"category": "other", "upload_id": id}), + ) + .await + .unwrap(); + let att = filed["attachment_id"].as_u64().unwrap(); + assert_eq!( + filed["sha256"], + json!(ledger_store::journal::sha256_hex(&bytes)) + ); + let e = call( + &c, + "books.file_document", + json!({"category": "other", "upload_id": id}), + ) + .await + .unwrap_err(); + assert!(e.contains("no upload"), "{e}"); + let meta = call(&c, "books.get_receipt", json!({"attachment_id": att})) + .await + .unwrap(); + assert_eq!(meta["file_name"], json!("big receipt.pdf")); + assert_eq!(meta["mime_type"], json!("application/pdf")); + assert_eq!(meta["size_bytes"], json!(bytes.len())); + assert!(meta.get("content_base64").is_none_or(Value::is_null)); + // Too large to come back inline: the command that saves a copy. + let big = call( + &c, + "books.get_receipt", + json!({"attachment_id": att, "view": "bytes"}), + ) + .await + .unwrap(); + assert!(big.get("content_base64").is_none_or(Value::is_null)); + assert_eq!( + big["download"], + json!(format!( + "wanderbooks download --book acme --token test {att} --to " + )) + ); + // The same bytes again: the attachment that holds them, no upload. + let again = call( + &c, + "books.begin_upload", + json!({"sha256": ledger_store::journal::sha256_hex(&bytes), "size": bytes.len(), + "mime_type": "application/pdf", "file_name": "copy.pdf", "purpose": "receipt"}), + ) + .await + .unwrap(); + assert_eq!(again["existing_attachment_id"], json!(att)); + assert!(again.get("upload_id").is_none_or(Value::is_null)); + // Nothing is left behind in the uploads' folder. + let left: Vec<_> = std::fs::read_dir(home.book_dir("acme").join("uploads")) + .unwrap() + .collect(); + assert!(left.is_empty(), "{left:?}"); + drop((c, c2)); + assert!(server.close().success()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_call_that_fails_leaves_its_upload_for_the_next() { + let home = Home::new("upload-again"); + let server = Server::start(&home); + home.create("acme", "USD"); + let c = connect(&home, "acme").await.unwrap(); + let token = home.token("acme"); + chart(&c).await; + let csv = b"Date,Description,Amount\n2026-03-02,Coffee,-4.50\n".to_vec(); + let id = begin(&c, &csv, "bank.csv", "import").await; + let (status, _, body) = put(&home, &token, &id, &csv).await; + assert_eq!(status, 200, "{body}"); + // Refused after the file was taken (no such account): given back. + let e = call( + &c, + "books.import_bank_csv", + json!({"upload_id": id, "bank_account_code": "1999", "default_offset_code": "6000"}), + ) + .await + .unwrap_err(); + assert!(e.contains("unknown bank account"), "{e}"); + let done = call( + &c, + "books.import_bank_csv", + json!({"upload_id": id, "bank_account_code": "1000", "default_offset_code": "6000"}), + ) + .await + .unwrap(); + assert_eq!(done["count"], json!(1)); + let e = call( + &c, + "books.import_bank_csv", + json!({"upload_id": id, "bank_account_code": "1000", "default_offset_code": "6000"}), + ) + .await + .unwrap_err(); + assert!(e.contains("no upload"), "{e}"); + // A chart refused after it was taken is given back; filed, used up. + let chart = b"code,name,type,parent_code\n7000,Rent,expense,\n".to_vec(); + let id = begin(&c, &chart, "chart.csv", "import").await; + assert_eq!(put(&home, &token, &id, &chart).await.0, 200); + let bad = b"code,name,type,parent_code\n7000,Rent,nonsense,\n".to_vec(); + let bad_id = begin(&c, &bad, "bad.csv", "import").await; + assert_eq!(put(&home, &token, &bad_id, &bad).await.0, 200); + let e = call(&c, "books.import_chart_csv", json!({"upload_id": bad_id})) + .await + .unwrap_err(); + assert!(e.contains("nonsense"), "{e}"); + let e = call(&c, "books.import_chart_csv", json!({"upload_id": bad_id})) + .await + .unwrap_err(); + assert!(e.contains("nonsense"), "given back: {e}"); + call(&c, "books.import_chart_csv", json!({"upload_id": id})) + .await + .unwrap(); + let e = call(&c, "books.import_chart_csv", json!({"upload_id": id})) + .await + .unwrap_err(); + assert!(e.contains("no upload"), "{e}"); + // A receipt attached is used up too. + let receipt = pdf("receipt", 900); + let id = begin(&c, &receipt, "r.pdf", "receipt").await; + assert_eq!(put(&home, &token, &id, &receipt).await.0, 200); + call(&c, "books.attach_receipt", json!({"upload_id": id})) + .await + .unwrap(); + let e = call(&c, "books.attach_receipt", json!({"upload_id": id})) + .await + .unwrap_err(); + assert!(e.contains("no upload"), "{e}"); + // Filed only by a tool of its purpose; the wrong tool leaves it. + let doc = pdf("purpose", 500); + let id = begin(&c, &doc, "doc.pdf", "import").await; + let (status, _, body) = put(&home, &token, &id, &doc).await; + assert_eq!(status, 200, "{body}"); + let e = call( + &c, + "books.file_document", + json!({"category": "other", "upload_id": id}), + ) + .await + .unwrap_err(); + assert!(e.contains("begun for import"), "{e}"); + let e = call(&c, "books.import_chart_csv", json!({"upload_id": id})) + .await + .unwrap_err(); + assert!(e.contains("isn't UTF-8"), "{e}"); + // The import refused the bytes (not a chart) after taking them: still + // there for the next call, which is refused for its purpose again. + let e = call(&c, "books.attach_receipt", json!({"upload_id": id})) + .await + .unwrap_err(); + assert!(e.contains("begun for import"), "{e}"); + let e = call( + &c, + "books.begin_upload", + json!({"sha256": ledger_store::journal::sha256_hex(&doc), "size": doc.len(), + "mime_type": "application/pdf", "file_name": "x", "purpose": "anything"}), + ) + .await + .unwrap_err(); + assert!(e.contains("purpose is"), "{e}"); + drop(c); + assert!(server.close().success()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn stored_bytes_are_named_only_where_they_can_be_linked() { + let home = Home::new("stored"); + let server = Server::start(&home); + home.create("acme", "USD"); + let c = connect(&home, "acme").await.unwrap(); + let csv = b"code,name,type,parent_code\n1000,Cash,asset,\n".to_vec(); + let filed = call( + &c, + "books.file_document", + json!({"category": "import_source", "file_name": "chart.csv", "mime_type": "text/csv", + "content_base64": b64(&csv), "sha256": ledger_store::journal::sha256_hex(&csv)}), + ) + .await + .unwrap(); + let att = filed["attachment_id"].as_u64().unwrap(); + let begin_for = |purpose: &'static str| { + let c = &c; + let csv = csv.clone(); + async move { + call( + c, + "books.begin_upload", + json!({"sha256": ledger_store::journal::sha256_hex(&csv), "size": csv.len(), + "mime_type": "text/csv", "file_name": "chart.csv", "purpose": purpose}), + ) + .await + .unwrap() + } + }; + // A document: the attachment that holds it. An import needs its bytes. + assert_eq!( + begin_for("document").await["existing_attachment_id"], + json!(att) + ); + let import = begin_for("import").await; + assert!(import["upload_id"].as_str().unwrap().starts_with("up_")); + assert!( + import + .get("existing_attachment_id") + .is_none_or(Value::is_null) + ); + // A withdrawn attachment can't be linked: never named. + call( + &c, + "books.withdraw_attachment", + json!({"attachment_id": att, "reason": "filed twice"}), + ) + .await + .unwrap(); + let again = begin_for("document").await; + assert!(again["upload_id"].as_str().unwrap().starts_with("up_")); + assert!( + again + .get("existing_attachment_id") + .is_none_or(Value::is_null) + ); + drop(c); + assert!(server.close().success()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn bytes_given_inline_are_small_and_carry_their_hash() { + let home = Home::new("inline"); + let server = Server::start(&home); + home.create("acme", "USD"); + let c = connect(&home, "acme").await.unwrap(); + let small = pdf("small", 1000); + let sha = ledger_store::journal::sha256_hex(&small); + let file = |extra: Value| { + let mut args = json!({"category": "other", "file_name": "s.pdf", + "mime_type": "application/pdf"}); + args.as_object_mut() + .unwrap() + .extend(extra.as_object().unwrap().clone()); + args + }; + for (extra, why) in [ + (json!({"content_base64": b64(&small)}), "sha256"), + ( + json!({"content_base64": b64(&small), "sha256": ledger_store::journal::sha256_hex(b"x")}), + "lost on the way", + ), + ( + json!({"content_base64": "not base64!", "sha256": sha}), + "not valid base64", + ), + ( + json!({"content_base64": b64(&small), "sha256": sha, "upload_id": "up_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}), + "not both", + ), + (json!({}), "missing"), + ] { + let e = call(&c, "books.file_document", file(extra)) + .await + .unwrap_err(); + assert!(e.contains(why), "{why}: {e}"); + } + // At most 256 KiB inline (the call's own limit is 1 MiB of base64). + let over = pdf("over", 256 * 1024 + 1); + let e = call( + &c, + "books.file_document", + file(json!({"content_base64": b64(&over), + "sha256": ledger_store::journal::sha256_hex(&over)})), + ) + .await + .unwrap_err(); + assert!(e.contains("upload it with books.begin_upload"), "{e}"); + // Text longer than the most bytes' base64 is refused unread. + let long = "!".repeat((256 * 1024usize).div_ceil(3) * 4 + 4); + let e = call( + &c, + "books.file_document", + file(json!({"content_base64": long, "sha256": sha})), + ) + .await + .unwrap_err(); + assert!(e.contains("inline is at most"), "{e}"); + // A plain name and a real type. + for (name, mime, why) in [ + ("../escape.pdf", "application/pdf", "no folder"), + ("a\\b.pdf", "application/pdf", "no folder"), + ("rcpt\u{202e}fdp.exe", "application/pdf", "invisible"), + ("a\nb.pdf", "application/pdf", "control"), + (" ", "application/pdf", "name is needed"), + ("ok.pdf", "pdf", "MIME type"), + ("ok.pdf", "text/plain\r\nX-Evil: 1", "MIME type"), + ] { + let e = call( + &c, + "books.file_document", + json!({"category": "other", "file_name": name, "mime_type": mime, + "content_base64": b64(&small), "sha256": sha}), + ) + .await + .unwrap_err(); + assert!(e.contains(why), "{name:?} {mime:?}: {e}"); + } + let e = call( + &c, + "books.file_document", + file(json!({"file_name": "x".repeat(256), "content_base64": b64(&small), "sha256": sha})), + ) + .await + .unwrap_err(); + assert!(e.contains("at most 255 bytes"), "{e}"); + let at = pdf("at", 256 * 1024); + let filed = call( + &c, + "books.file_document", + file(json!({"content_base64": b64(&at), + "sha256": ledger_store::journal::sha256_hex(&at).to_uppercase()})), + ) + .await + .unwrap(); + let att = filed["attachment_id"].as_u64().unwrap(); + let back = call( + &c, + "books.get_receipt", + json!({"attachment_id": att, "view": "bytes"}), + ) + .await + .unwrap(); + assert_eq!(back["content_base64"], json!(b64(&at))); + assert!(back.get("download").is_none_or(Value::is_null)); + let e = call( + &c, + "books.get_receipt", + json!({"attachment_id": att, "view": "everything"}), + ) + .await + .unwrap_err(); + assert!(e.contains("meta") && e.contains("bytes"), "{e}"); + drop(c); + assert!(server.close().success()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_download_is_the_stored_bytes_for_a_reader_and_is_recorded() { + let home = Home::new("download"); + let server = Server::start(&home); + home.create("acme", "USD"); + let c = connect(&home, "acme").await.unwrap(); + let token = home.token("acme"); + let bytes = pdf("down", 2000); + let filed = call( + &c, + "books.file_document", + json!({"category": "other", "file_name": "Résumé \"final\".pdf", + "mime_type": "application/pdf", "content_base64": b64(&bytes), + "sha256": ledger_store::journal::sha256_hex(&bytes)}), + ) + .await + .unwrap(); + let att = filed["attachment_id"].as_u64().unwrap(); + let get = |bearer: Option, what: String| { + let http = mcp_http(&home); + async move { + let mut r = http.get(format!("http://localhost/b/acme/a/{what}")); + if let Some(b) = bearer { + r = r.bearer_auth(b); + } + r.send().await.unwrap() + } + }; + let r = get(Some(token.clone()), att.to_string()).await; + assert_eq!(r.status().as_u16(), 200); + let h = r.headers().clone(); + assert_eq!(h["content-type"], "application/pdf"); + assert_eq!(h["x-content-type-options"], "nosniff"); + assert_eq!( + h["content-disposition"], + "attachment; filename=\"R_sum_ _final_.pdf\"; filename*=UTF-8''R%C3%A9sum%C3%A9%20%22final%22.pdf" + ); + let digest = ::digest(&bytes); + assert_eq!( + h["repr-digest"], + format!("sha-256=:{}:", b64(&digest)).as_str() + ); + assert_eq!(r.bytes().await.unwrap().to_vec(), bytes); + // A token without books:read gets nothing; nor does no token. + let writer = home.token_with("acme", "writer", &["write"]); + let r = get(Some(writer), att.to_string()).await; + assert_eq!(r.status().as_u16(), 403); + assert!(r.text().await.unwrap().contains("books:read")); + assert_eq!(get(None, att.to_string()).await.status().as_u16(), 401); + // GET only: a HEAD reads nothing and records nothing. + let head = mcp_http(&home) + .head(format!("http://localhost/b/acme/a/{att}")) + .bearer_auth(&token) + .send() + .await + .unwrap(); + assert_eq!(head.status().as_u16(), 405); + assert_eq!(head.headers()["allow"], "GET"); + for missing in [ + "999".to_string(), + "x".to_string(), + "-1".to_string(), + format!("+{att}"), + format!("0{att}"), + ] { + let r = get(Some(token.clone()), missing).await; + assert_eq!(r.status().as_u16(), 404); + home.assert_hidden(&r.text().await.unwrap()); + } + // Each is in the access log, refusals too. + let log = call( + &c, + "books.access_log", + json!({"action": "download", "limit": 10}), + ) + .await + .unwrap(); + let outcomes: Vec = log["records"] + .as_array() + .unwrap() + .iter() + .map(|r| r["result"].to_string()) + .collect(); + assert_eq!(outcomes.len(), 8, "{outcomes:?}"); + // A TIN in the path is masked in the log, whole. + let r = get(Some(token.clone()), "123-45-6789".to_string()).await; + assert_eq!(r.status().as_u16(), 404); + let log = call( + &c, + "books.access_log", + json!({"action": "download", "limit": 1}), + ) + .await + .unwrap(); + let subject = log["records"][0]["subject"].as_str().unwrap().to_string(); + assert!( + !subject.contains("6789") && !subject.contains("123-45"), + "{subject}" + ); + assert!( + outcomes + .iter() + .filter(|o| o.contains("no_such_attachment")) + .count() + == 5 + ); + assert!(outcomes.iter().any(|o| o.contains("insufficient_scope"))); + drop(c); + assert!(server.close().success()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_w9_and_a_file_holding_a_tin_go_by_upload_only_where_they_belong() { + let home = Home::new("w9-upload"); + let server = Server::start(&home); + home.create("acme", "USD"); + let c = connect(&home, "acme").await.unwrap(); + let token = home.token("acme"); + call( + &c, + "books.add_party", + json!({"name": "Payee", "roles": ["contractor"]}), + ) + .await + .unwrap(); + call( + &c, + "books.set_tin", + json!({"party": "Payee", "tin": "620-37-1111", "tin_type": "ssn"}), + ) + .await + .unwrap(); + // A file holding a TIN the vault keeps is refused, upload or not. + let notes = b"Payee notes: SSN 620-37-1111".to_vec(); + let id = begin(&c, ¬es, "notes.txt", "document").await; + assert_eq!(put(&home, &token, &id, ¬es).await.0, 200); + let e = call( + &c, + "books.file_document", + json!({"category": "other", "upload_id": id, "mime_type": "text/plain"}), + ) + .await + .unwrap_err(); + assert!(e.contains("TIN vault"), "{e}"); + // A W-9 goes only to the vault. + let w9 = b"%PDF-1.4 Form W-9 Payee 620-37-1111".to_vec(); + let id = begin(&c, &w9, "payee-w9.pdf", "w9").await; + assert_eq!(put(&home, &token, &id, &w9).await.0, 200); + for (tool, args) in [ + ("books.attach_receipt", json!({"upload_id": id})), + ( + "books.file_document", + json!({"category": "tax", "upload_id": id}), + ), + ] { + let e = call(&c, tool, args).await.unwrap_err(); + assert!(e.contains("begun for w9"), "{tool}: {e}"); + } + let w9_args = |party: &str| { + json!({"party": party, "received_on": "2026-01-05", + "entity_type": "individual/sole proprietor", "tin_last4": "1111", + "w9_upload_id": id}) + }; + // Refused after it was taken: given back. + let e = call(&c, "books.set_w9", w9_args("Nobody")) + .await + .unwrap_err(); + assert!(!e.contains("upload"), "{e}"); + let set = call(&c, "books.set_w9", w9_args("Payee")).await.unwrap(); + assert_eq!(set["w9"]["in_vault"], json!(true), "{set}"); + let e = call(&c, "books.set_w9", w9_args("Payee")) + .await + .unwrap_err(); + assert!(e.contains("no upload"), "{e}"); + let out = home.dir.path().join("payee copy.pdf"); + let got = call( + &c, + "books.get_w9", + json!({"party": "Payee", "save_to": out.to_str().unwrap()}), + ) + .await + .unwrap(); + assert_eq!(got["file_name"], json!("payee-w9.pdf")); + assert_eq!(std::fs::read(&out).unwrap(), w9); + // The W-9's upload is gone; the refused file's waits to lapse. + let left: Vec = std::fs::read_dir(home.book_dir("acme").join("uploads")) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + assert!(left.iter().all(|n| !n.starts_with(&id)), "{left:?}"); + drop(c); + assert!(server.close().success()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn lapsed_uploads_are_swept_while_the_server_runs() { + let home = Home::new("sweep"); + let server = Server::start(&home); + home.create("acme", "USD"); + let c = connect(&home, "acme").await.unwrap(); + let token = home.token("acme"); + let bytes = pdf("lapsing", 2000); + let id = begin(&c, &bytes, "w9.pdf", "w9").await; + assert_eq!(put(&home, &token, &id, &bytes).await.0, 200); + let folder = home.book_dir("acme").join("uploads"); + // Its time runs out (the grant aged on the disk), and nothing else + // happens: the server sweeps it anyway. + let json = folder.join(format!("{id}.json")); + let mut grant: Value = serde_json::from_slice(&std::fs::read(&json).unwrap()).unwrap(); + grant["expires"] = json!(0); + std::fs::write(&json, grant.to_string()).unwrap(); + let deadline = Instant::now() + Duration::from_secs(25); + while std::fs::read_dir(&folder).unwrap().count() > 0 { + assert!(Instant::now() < deadline, "the lapsed upload wasn't swept"); + tokio::time::sleep(Duration::from_millis(200)).await; + } + drop(c); + assert!(server.close().success()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_mcp_request_is_at_most_a_mebibyte() { + let home = Home::new("body"); + let server = Server::start(&home); + home.create("acme", "USD"); + let http = mcp_http_for(&home, "acme"); + let session = raw_session(&http, "acme").await; + let post = |len: usize| { + // A call whose arguments pad it to `len` bytes exactly. + let head = r#"{"jsonrpc":"2.0","id":5,"method":"tools/call","params":{"name":"books.chart","arguments":{"pad":""#; + let tail = r#""}}}"#; + let body = format!("{head}{}{tail}", "a".repeat(len - head.len() - tail.len())); + assert_eq!(body.len(), len); + http.post("http://localhost/b/acme/mcp") + .header("content-type", "application/json") + .header("accept", "application/json, text/event-stream") + .header("mcp-session-id", &session) + .header("mcp-protocol-version", "2025-06-18") + .body(body) + .send() + }; + let max = ledger_mcp::local::MAX_BODY; + assert_eq!(post(max + 1).await.unwrap().status().as_u16(), 413); + let at = post(max).await.unwrap(); + assert_ne!(at.status().as_u16(), 413, "{:?}", at.text().await); + drop(http); + assert!(server.close().success()); +} diff --git a/crates/ledger-cli/tests/one_executable.rs b/crates/ledger-cli/tests/one_executable.rs index 9e38d68..d31dfff 100644 --- a/crates/ledger-cli/tests/one_executable.rs +++ b/crates/ledger-cli/tests/one_executable.rs @@ -35,16 +35,10 @@ fn dev_stdio_is_the_books_server() { .tempdir() .unwrap(); let data = dir.path().join("books").join("books-data"); - // An import root may not be inside the books dir (the data dir's - // parent), so it is a sibling of it. - let inbox = dir.path().join("inbox"); - std::fs::create_dir(&inbox).unwrap(); let mut child = wanderbooks() .arg("dev-stdio") .arg("--data-dir") .arg(&data) - .arg("--import-root") - .arg(&inbox) .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::null()) diff --git a/crates/ledger-cli/tests/open_while_held.rs b/crates/ledger-cli/tests/open_while_held.rs index 37b4e42..a57de68 100644 --- a/crates/ledger-cli/tests/open_while_held.rs +++ b/crates/ledger-cli/tests/open_while_held.rs @@ -383,7 +383,8 @@ fn post_one(a: &mut Agent, n: u64) -> Result { a.call( "books.attach_receipt", json!({"entry_id": id, "file_name": format!("r{n}.txt"), - "mime_type": "text/plain", "content_base64": "cmVjZWlwdA=="}), + "mime_type": "text/plain", "content_base64": "cmVjZWlwdA==", + "sha256": "6f32860910ca0fb2a20c7fda143666b09dbf8db5238195c90a586fb542ff0cad"}), )?; a.call( "books.post_entry", @@ -1354,11 +1355,10 @@ fn a_message_too_large_is_refused_and_the_session_goes_on() { a.initialize(); { let stdin = a.run.child.stdin.as_mut().unwrap(); + // One byte over the limit (1 MiB), its newline aside. + let max = ledger_mcp::local::MAX_BODY; stdin.write_all(b"{\"x\":\"").unwrap(); - let chunk = vec![b'a'; 1024 * 1024]; - for _ in 0..65 { - stdin.write_all(&chunk).unwrap(); - } + stdin.write_all(&vec![b'a'; max + 1 - 8]).unwrap(); stdin.write_all(b"\"}\n").unwrap(); stdin.flush().unwrap(); } @@ -1427,10 +1427,11 @@ fn malformed_and_boundary_input_never_ends_the_session() { assert!(line.contains("-32700"), "{line}"); let pong = a.request("ping", json!({}), 20); assert!(pong.get("result").is_some(), "{pong}"); - // Exactly 64 MiB of message (a JSON string padded with spaces), then - // a ping: the big one is the server's to refuse, the ping answered. + // Exactly the limit, 1 MiB of message (a JSON string padded with + // spaces), then a ping: the big one is the server's to refuse, the + // ping answered. { - let max = 64 * 1024 * 1024; + let max = 1024 * 1024; let head = b"{\"jsonrpc\":\"2.0\",\"method\":\"notifications/x\",\"params\":{\"pad\":\""; let tail = b"\"}}"; let stdin = a.run.child.stdin.as_mut().unwrap(); @@ -2050,3 +2051,281 @@ fn the_token_commands_and_the_bridge() { assert!(err.contains("isn't a scope"), "{err}"); assert!(b.end().is_some()); } + +/// Run `command` (as books.begin_upload or books.get_receipt gave it, +/// with its placeholder cut off) with `more` words after it: its output. +fn run_given( + home: &Home, + command: &str, + placeholder: &str, + more: &[&std::ffi::OsStr], +) -> std::process::Output { + let words: Vec<&str> = command + .strip_suffix(placeholder) + .unwrap_or_else(|| panic!("{command}")) + .split(' ') + .collect(); + assert_eq!(words[0], "wanderbooks", "{command}"); + home.cmd(words[1]) + .args(&words[2..]) + .args(more) + .stdin(Stdio::null()) + .output() + .unwrap() +} + +#[test] +fn an_agent_sends_and_saves_files_with_the_commands_it_is_given() { + let home = Home::new("files"); + home.create("acme"); + let mut a = Agent::start(&home, "acme"); + a.initialize(); + let folder = home.dir.path().join("Downloads"); + std::fs::create_dir(&folder).unwrap(); + let mut bytes = b"%PDF-1.4 a big receipt ".to_vec(); + bytes.resize(400 * 1024, b'x'); + let file = folder.join("big receipt.pdf"); + std::fs::write(&file, &bytes).unwrap(); + let begin = |a: &mut Agent| { + a.call( + "books.begin_upload", + json!({"sha256": ledger_store::journal::sha256_hex(&bytes), "size": bytes.len(), + "mime_type": "application/pdf", "file_name": "big receipt.pdf", + "purpose": "receipt"}), + ) + .unwrap() + }; + let begun = begin(&mut a); + let upload = begun["upload_id"].as_str().unwrap().to_string(); + let command = begun["command"].as_str().unwrap().to_string(); + let quiet = |out: &std::process::Output| { + let text = format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + assert!(!text.contains(&home.root().display().to_string()), "{text}"); + text + }; + // Another file than the one declared: refused, nothing kept. + let other = folder.join("other.pdf"); + std::fs::write(&other, b"%PDF-1.4 not it").unwrap(); + let out = run_given(&home, &command, " ", &[other.as_os_str()]); + assert!(!out.status.success()); + assert!(quiet(&out).contains("not the"), "{}", quiet(&out)); + // A file of exactly the largest size isn't refused for its size. + let edge = folder.join("edge.pdf"); + std::fs::File::create(&edge) + .unwrap() + .set_len(25 * 1024 * 1024) + .unwrap(); + let out = run_given(&home, &command, " ", &[edge.as_os_str()]); + assert!(!out.status.success()); + assert!( + quiet(&out).contains("more than the") && !quiet(&out).contains("at most"), + "{}", + quiet(&out) + ); + std::fs::remove_file(&edge).unwrap(); + // A file larger than any upload is refused before it is read. + let huge = folder.join("huge.pdf"); + std::fs::File::create(&huge) + .unwrap() + .set_len(25 * 1024 * 1024 + 1) + .unwrap(); + let out = run_given(&home, &command, " ", &[huge.as_os_str()]); + assert!(!out.status.success()); + assert!(quiet(&out).contains("at most"), "{}", quiet(&out)); + // With no token kept for the name given: what the human does. + let out = home + .cmd("upload") + .args(["--book", "acme", "--token", "nobody", &upload]) + .arg(&file) + .stdin(Stdio::null()) + .output() + .unwrap(); + assert!(!out.status.success()); + assert!( + quiet(&out).contains("wanderbooks token create"), + "{}", + quiet(&out) + ); + // A path that looks like a flag goes after the command's `--`. + let dashed = folder.join("-r.pdf"); + std::fs::copy(&file, &dashed).unwrap(); + let out = home + .cmd("upload") + .args(["--book", "acme", "not-an-id", "--"]) + .arg(&dashed) + .stdin(Stdio::null()) + .output() + .unwrap(); + assert_eq!(out.status.code(), Some(2), "{out:?}"); + assert!( + quiet(&out).contains("isn't an upload's id"), + "{}", + quiet(&out) + ); + // The file itself goes (by its copy whose name starts with `-`). + let out = run_given(&home, &command, " ", &[dashed.as_os_str()]); + assert!(out.status.success(), "{}", quiet(&out)); + assert!(quiet(&out).contains(&format!("Sent {} bytes", bytes.len()))); + let filed = a + .call( + "books.file_document", + json!({"category": "other", "upload_id": upload}), + ) + .unwrap(); + let att = filed["attachment_id"].as_u64().unwrap(); + // Too big to come back inline: the command saves a copy, beside any + // file of its name, never over it. + let back = a + .call( + "books.get_receipt", + json!({"attachment_id": att, "view": "bytes"}), + ) + .unwrap(); + let download = back["download"].as_str().unwrap().to_string(); + for n in 1..=2 { + let out = run_given( + &home, + &download, + " --to ", + &["--to".as_ref(), folder.as_os_str()], + ); + assert!(out.status.success(), "{}", quiet(&out)); + let saved = folder.join(format!("big receipt ({n}).pdf")); + assert_eq!(std::fs::read(&saved).unwrap(), bytes); + assert!(quiet(&out).contains(&saved.display().to_string())); + } + assert_eq!(std::fs::read(&file).unwrap(), bytes, "never overwritten"); + let out = run_given( + &home, + &download, + " --to ", + &["--to".as_ref(), file.as_os_str()], + ); + assert!(!out.status.success()); + assert!(quiet(&out).contains("isn't a folder")); + let out = home + .cmd("download") + .args(["--book", "acme", "999", "--to"]) + .arg(&folder) + .stdin(Stdio::null()) + .output() + .unwrap(); + assert!(!out.status.success()); + quiet(&out); + assert_eq!( + std::fs::read_dir(&folder).unwrap().count(), + 6, + "nothing else saved" + ); + a.end(); +} + +#[test] +fn the_commands_given_run_as_the_agents_own_token() { + let home = Home::new("files-token"); + home.create("acme"); + let out = home + .cmd("token") + .args(["create", "--book", "acme", "--name", "bookkeeper"]) + .stdin(Stdio::null()) + .output() + .unwrap(); + assert!(out.status.success(), "{out:?}"); + let mut c = home.cmd("mcp"); + c.args(["--book", "acme", "--token", "bookkeeper"]); + let mut a = Agent::from(c); + a.initialize(); + let folder = home.dir.path().join("Downloads"); + std::fs::create_dir(&folder).unwrap(); + let mut bytes = b"%PDF-1.4 kept by the bookkeeper ".to_vec(); + bytes.resize(300 * 1024, b'y'); + let file = folder.join("r.pdf"); + std::fs::write(&file, &bytes).unwrap(); + let begun = a + .call( + "books.begin_upload", + json!({"sha256": ledger_store::journal::sha256_hex(&bytes), "size": bytes.len(), + "mime_type": "application/pdf", "file_name": "r.pdf", "purpose": "receipt"}), + ) + .unwrap(); + let command = begun["command"].as_str().unwrap().to_string(); + assert!(command.contains(" --token bookkeeper "), "{command}"); + let out = run_given(&home, &command, " ", &[file.as_os_str()]); + assert!(out.status.success(), "{out:?}"); + let filed = a + .call( + "books.file_document", + json!({"category": "other", "upload_id": begun["upload_id"]}), + ) + .unwrap(); + let back = a + .call( + "books.get_receipt", + json!({"attachment_id": filed["attachment_id"], "view": "bytes"}), + ) + .unwrap(); + let download = back["download"].as_str().unwrap().to_string(); + assert!(download.contains(" --token bookkeeper "), "{download}"); + let out = run_given( + &home, + &download, + " --to ", + &["--to".as_ref(), folder.as_os_str()], + ); + assert!(out.status.success(), "{out:?}"); + assert_eq!(std::fs::read(folder.join("r (1).pdf")).unwrap(), bytes); + // A file whose name isn't text goes, and comes back into a folder + // whose name isn't text. + use std::os::unix::ffi::OsStrExt; + let odd = folder.join(std::ffi::OsStr::from_bytes(b"r\xff.pdf")); + let mut other = bytes.clone(); + other[40] = b'z'; + std::fs::write(&odd, &other).unwrap(); + let begun = a + .call( + "books.begin_upload", + json!({"sha256": ledger_store::journal::sha256_hex(&other), "size": other.len(), + "mime_type": "application/pdf", "file_name": "r2.pdf", "purpose": "receipt"}), + ) + .unwrap(); + let out = run_given( + &home, + begun["command"].as_str().unwrap(), + " ", + &[odd.as_os_str()], + ); + assert!(out.status.success(), "{out:?}"); + let filed = a + .call( + "books.file_document", + json!({"category": "other", "upload_id": begun["upload_id"]}), + ) + .unwrap(); + let odd_folder = folder.join(std::ffi::OsStr::from_bytes(b"out\xfe")); + std::fs::create_dir(&odd_folder).unwrap(); + let out = home + .cmd("download") + .args(["--book", "acme", "--token", "bookkeeper"]) + .arg(filed["attachment_id"].to_string()) + .arg("--to") + .arg(&odd_folder) + .stdin(Stdio::null()) + .output() + .unwrap(); + assert!(out.status.success(), "{out:?}"); + assert_eq!(std::fs::read(odd_folder.join("r2.pdf")).unwrap(), other); + // Anywhere else, a word that isn't text is refused plainly. + let out = home + .cmd("books") + .arg(std::ffi::OsStr::from_bytes(b"\xff")) + .stdin(Stdio::null()) + .output() + .unwrap(); + assert_eq!(out.status.code(), Some(2), "{out:?}"); + assert!(String::from_utf8_lossy(&out.stderr).contains("isn't UTF-8")); + a.end(); +} diff --git a/crates/ledger-host/Cargo.toml b/crates/ledger-host/Cargo.toml index 85dd6df..ebd75f7 100644 --- a/crates/ledger-host/Cargo.toml +++ b/crates/ledger-host/Cargo.toml @@ -15,6 +15,8 @@ serde_json = "1" tokio = { version = "1", features = ["rt-multi-thread", "macros", "io-util", "io-std", "sync", "net", "signal", "time", "process"] } reqwest = "0.13" base64 = "0.22" +# `wanderbooks download` checks what arrived against its Repr-Digest. +sha2 = "0.10" tokio-util = { version = "0.7", features = ["rt"] } # Agents' tokens are kept in the OS keyring (plan 3.3, 3.5): the Secret # Service on Linux. macOS and Windows stores come with P11. diff --git a/crates/ledger-host/src/bridge.rs b/crates/ledger-host/src/bridge.rs index 45e03d4..28d9b58 100644 --- a/crates/ledger-host/src/bridge.rs +++ b/crates/ledger-host/src/bridge.rs @@ -59,8 +59,8 @@ use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt}; use tokio::sync::{Notify, mpsc, watch}; use tokio::time::Instant; -/// The longest message the agent may send (receipts ride in calls until -/// uploads have their own route). +/// The longest message the agent may send (a file bigger than a call +/// takes inline goes by an upload). const MAX_MESSAGE: usize = ledger_mcp::local::MAX_BODY; /// The bridge's own request ids upstream start so. const OWN_ID: &str = "wanderbooks-bridge-"; diff --git a/crates/ledger-host/src/commands.rs b/crates/ledger-host/src/commands.rs index ac33e04..93fbc5c 100644 --- a/crates/ledger-host/src/commands.rs +++ b/crates/ledger-host/src/commands.rs @@ -1158,10 +1158,676 @@ pub fn passphrase( Ok(()) } +/// A request to a book over the server's socket, with the agent's token +/// from the keyring, the books held open meanwhile: its status, the +/// headers wanted, and its body. +fn on_the_socket( + common: &Common, + book: &str, + token_name: &str, + request: impl FnOnce(&reqwest::Client, String) -> reqwest::RequestBuilder, +) -> Result<(u16, reqwest::header::HeaderMap, Vec), String> { + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .map_err(|e| e.to_string())?; + let outcome = rt.block_on(async { + let mut opener = Opener::new( + common.root.as_deref(), + common.knobs, + Starter::Direct, + Kind::Cli, + None, + )?; + let lease = open(&mut opener).await.map_err(|e| match e { + HoldError::ClosedByOwner => { + "the books were closed just now (Close the books): nothing was done".to_string() + } + HoldError::Failed(why) => why, + })?; + let run_dir = opener.run_dir().to_path_buf(); + let root = run_dir.parent().unwrap_or(&run_dir).to_path_buf(); + let account = crate::tokens::Account::new(&root, book, token_name); + let keeper = crate::tokens::keeper(&run_dir, &common.knobs); + let token = tokio::task::spawn_blocking(move || keeper.get(&account)) + .await + .map_err(|e| e.to_string())?? + .ok_or_else(|| { + format!( + "this computer has no token for the books {book} (named {token_name:?}): the human makes one with `wanderbooks token create --book {book}`" + ) + })?; + let token = zeroize::Zeroizing::new(token); + let socket = ledger_store::root::SocketPath::new(&run_dir, ledger_mcp::local::MCP_SOCKET) + .map_err(|e| format!("can't reach the books server: {e}"))?; + let http = reqwest::Client::builder() + .unix_socket(socket.path()) + .no_proxy() + .build() + .map_err(|e| e.to_string())?; + let response = request(&http, format!("http://localhost/b/{book}")) + .bearer_auth(token.as_str()) + .send() + .await + .map_err(|e| format!("the books server didn't answer: {e}"))?; + let status = response.status().as_u16(); + let headers = response.headers().clone(); + let body = response + .bytes() + .await + .map_err(|e| format!("the books server's answer was cut off: {e}"))? + .to_vec(); + drop(lease); + Ok::<_, String>((status, headers, body)) + }); + rt.shutdown_timeout(Duration::from_secs(1)); + outcome +} + +/// The flags `upload` and `download` share: the books, the token's name, +/// and the words given. +struct FileArgs { + common: Common, + book: String, + token: String, + words: Vec, + to: Option, +} + +fn file_args( + args: impl IntoIterator, + usage: &dyn Fn(), +) -> Result, Box> { + let mut common = Common::default(); + let mut book = None; + let mut token = crate::tokens::DEFAULT_NAME.to_string(); + let mut words = Vec::new(); + let mut to = None; + let mut args = args.into_iter(); + // Flags and their values are text; a file's path, or the folder + // `--to` names, may be any name the system takes. + let text = |v: std::ffi::OsString, what: &str| { + v.into_string() + .map_err(|_| format!("{what} isn't UTF-8 text")) + }; + while let Some(arg) = args.next() { + // After `--`, words only (a file whose name starts with `-`). + if arg == "--" { + words.extend(args.by_ref()); + break; + } + let Some(flag) = arg.to_str().filter(|a| a.starts_with('-')) else { + words.push(arg); + continue; + }; + let flag = flag.to_string(); + let mut value = |what: &str| args.next().ok_or_else(|| format!("{flag} needs {what}")); + match flag.as_str() { + "--root" => common.root = Some(PathBuf::from(value("a folder")?)), + "--book" => book = Some(text(value("an id")?, "--book's id")?), + "--token" => token = text(value("a name")?, "--token's name")?, + "--to" => to = Some(PathBuf::from(value("a folder")?)), + "--help" | "-h" => { + usage(); + return Ok(None); + } + f if f == FLAG => { + common.knobs = Knobs::parse(&text(value("a value")?, FLAG)?)?; + } + s => { + eprintln!("unknown flag: {s}"); + usage(); + return Err(exit(2)); + } + } + } + let Some(book) = book else { + eprintln!("error: which books? give --book ID"); + return Err(exit(2)); + }; + let book = ledger_store::root::BookId::parse(&book) + .map(|id| id.to_string()) + .map_err(|e| e.to_string())?; + Ok(Some(FileArgs { + common, + book, + token, + words, + to, + })) +} + +/// `wanderbooks upload --book ID UPLOAD FILE`: send a file's bytes for an +/// upload begun with `books.begin_upload` (plan 4.7, D-28: the agent runs +/// it). +pub fn upload( + prog: &str, + args: impl IntoIterator, +) -> Result<(), Box> { + let usage = || { + eprintln!("usage: {prog} --book ID [--token NAME] UPLOAD [--] FILE"); + eprintln!(" Send FILE for UPLOAD (from books.begin_upload), with the agent's token."); + }; + let Some(a) = file_args(args, &usage)? else { + return Ok(()); + }; + let ([upload, file], None) = (a.words.as_slice(), &a.to) else { + usage(); + return Err(exit(2)); + }; + let (Some(upload), file) = ( + upload.to_str().filter(|u| ledger_store::uploads::is_id(u)), + std::path::Path::new(file).display(), + ) else { + eprintln!( + "error: that isn't an upload's id (books.begin_upload gives one: up_ and 32 letters)" + ); + return Err(exit(2)); + }; + let path = std::path::Path::new(&a.words[1]); + let fail = |why: String| -> Box { + eprintln!("error: {why}"); + exit(1) + }; + let max = ledger_store::uploads::MAX_UPLOAD; + match std::fs::metadata(path) { + Ok(m) if !m.is_file() => return Err(fail(format!("{file} isn't a file"))), + Ok(m) if m.len() > max => { + return Err(fail(format!( + "{file} is {} bytes; an upload is at most {max}", + m.len() + ))); + } + Ok(_) => {} + Err(e) => return Err(fail(format!("can't read {file}: {}", e.kind()))), + } + let bytes = + std::fs::read(path).map_err(|e| fail(format!("can't read {file}: {}", e.kind())))?; + let size = bytes.len(); + let (status, _, body) = on_the_socket(&a.common, &a.book, &a.token, |http, base| { + http.put(format!("{base}/u/{upload}")).body(bytes) + }) + .map_err(fail)?; + let said = String::from_utf8_lossy(&body).to_string(); + if status != 200 { + return Err(fail(said)); + } + println!("Sent {size} bytes for {upload}: name it (upload_id) in the tool that files it."); + Ok(()) +} + +/// The `Content-Disposition` parameters (RFC 6266): name and value, a +/// quoted value unquoted; a `;` or `=` inside quotes is part of the value. +fn disposition_params(value: &str) -> Vec<(String, String)> { + let mut params = Vec::new(); + let mut rest = value.split_once(';').map_or("", |(_, r)| r); + loop { + rest = rest.trim_start_matches([';', ' ', '\t']); + let Some((name, after)) = rest.split_once('=') else { + return params; + }; + let name = name.trim().to_ascii_lowercase(); + let after = after.trim_start(); + let (v, next) = if let Some(quoted) = after.strip_prefix('"') { + let mut v = String::new(); + let mut chars = quoted.char_indices(); + let mut end = quoted.len(); + while let Some((i, c)) = chars.next() { + match c { + '\\' => { + if let Some((_, e)) = chars.next() { + v.push(e); + } + } + '"' => { + end = i + 1; + break; + } + c => v.push(c), + } + } + (v, "ed[end.min(quoted.len())..]) + } else { + let end = after.find(';').unwrap_or(after.len()); + (after[..end].trim().to_string(), &after[end..]) + }; + params.push((name, v)); + rest = next; + } +} + +/// The longest name a download is saved under, in bytes (under every +/// file system's limit, with room for " (999)"). +const SAVED_NAME_MAX: usize = 200; + +/// The name a download is saved under: the server's (RFC 6266), its UTF-8 +/// `filename*` first; a plain file name, never hidden, nothing invisible +/// in it, at most SAVED_NAME_MAX bytes (its extension kept). +fn saved_name(headers: &reqwest::header::HeaderMap) -> Option { + let value = headers + .get(reqwest::header::CONTENT_DISPOSITION)? + .to_str() + .ok()?; + let params = disposition_params(value); + let utf8 = params + .iter() + .find(|(n, _)| n == "filename*") + .and_then(|(_, v)| v.strip_prefix("UTF-8''")) + .and_then(percent_decoded); + let plain = || { + params + .iter() + .find(|(n, _)| n == "filename") + .map(|(_, v)| v.clone()) + }; + plain_name(&utf8.or_else(plain)?) +} + +/// `name` made a plain, visible file name of at most SAVED_NAME_MAX +/// bytes: folders' separators and control characters become `_`, +/// invisible ones go, leading dots and spaces go; None if nothing's left. +fn plain_name(name: &str) -> Option { + let cleaned: String = name + .chars() + .filter(|c| !ledger_store::uploads::is_hidden_char(*c)) + .map(|c| { + if c == '/' || c == '\\' || c.is_control() { + '_' + } else { + c + } + }) + .collect(); + let cleaned = cleaned.trim().trim_start_matches(['.', ' ']).to_string(); + if cleaned.is_empty() { + return None; + } + if cleaned.len() <= SAVED_NAME_MAX { + return Some(cleaned); + } + // Too long: the stem cut on a character boundary, the extension kept + // (if it is short). + let (stem, ext) = match cleaned.rsplit_once('.') { + Some((s, e)) if !s.is_empty() && e.len() <= 16 => (s, format!(".{e}")), + _ => (cleaned.as_str(), String::new()), + }; + let mut cut = SAVED_NAME_MAX - ext.len(); + while !stem.is_char_boundary(cut) { + cut -= 1; + } + Some(format!("{}{ext}", stem[..cut].trim_end())) +} + +/// `%XX`-escaped UTF-8 (RFC 8187), or None when it isn't. +fn percent_decoded(text: &str) -> Option { + let mut bytes = Vec::with_capacity(text.len()); + let mut it = text.bytes(); + while let Some(b) = it.next() { + if b == b'%' { + let hex = [it.next()?, it.next()?]; + bytes.push(u8::from_str_radix(std::str::from_utf8(&hex).ok()?, 16).ok()?); + } else { + bytes.push(b); + } + } + String::from_utf8(bytes).ok() +} + +/// Whether `body` is what the server stored: its SHA-256 is the one the +/// server sent (RFC 9530 `Repr-Digest: sha-256=:BASE64:`). No digest, or +/// one that can't be read, is no match. +fn arrived_whole(headers: &reqwest::header::HeaderMap, body: &[u8]) -> bool { + let sent = headers + .get(ledger_mcp::local::DIGEST_HEADER) + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.trim().strip_prefix("sha-256=:")) + .and_then(|v| v.strip_suffix(':')) + .and_then(|v| base64::Engine::decode(&base64::engine::general_purpose::STANDARD, v).ok()); + sent.as_deref() == Some(&::digest(body)[..]) +} + +/// Save `body` in `folder` as `named`, or `named (1)`, `named (2)`... the +/// first free; `fallback` if the folder takes no such name. Written aside +/// (a hidden file) and linked in whole, so the name never holds part of +/// it; never over another file. The path it went to. +fn save_new( + folder: &std::path::Path, + named: &str, + fallback: &str, + body: &[u8], +) -> Result { + use std::io::Write; + let fail = |what: &std::path::Path, e: std::io::Error| { + format!("can't write {}: {}", what.display(), e.kind()) + }; + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.subsec_nanos()); + let aside = folder.join(format!(".wanderbooks-{}-{nanos}.part", std::process::id())); + let mut f = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&aside) + .map_err(|e| fail(folder, e))?; + if let Err(e) = f.write_all(body).and_then(|()| f.sync_all()) { + drop(f); + let _ = std::fs::remove_file(&aside); + return Err(fail(folder, e)); + } + drop(f); + let candidates = |name: &str| { + let (stem, ext) = match name.rsplit_once('.') { + Some((s, e)) if !s.is_empty() => (s.to_string(), format!(".{e}")), + _ => (name.to_string(), String::new()), + }; + let name = name.to_string(); + (0..1000).map(move |n| { + if n == 0 { + name.clone() + } else { + format!("{stem} ({n}){ext}") + } + }) + }; + let mut outcome = Err(format!("{} has no free name for it", folder.display())); + // Why the last name tried was refused, if not for being taken: what + // the human is told when no name is left (a full disk, not a name). + let mut last: Option = None; + 'names: for name in [named, fallback] { + for candidate in candidates(name) { + let path = folder.join(&candidate); + match std::fs::hard_link(&aside, &path) { + Ok(()) => { + outcome = Ok(path); + break 'names; + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {} + // A file system with no links (FAT): written in place, never + // over another file, removed if it can't be finished. + Err(e) + if matches!( + e.kind(), + std::io::ErrorKind::Unsupported | std::io::ErrorKind::PermissionDenied + ) => + { + match std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&path) + { + Ok(mut f) => { + if let Err(e) = f.write_all(body).and_then(|()| f.sync_all()) { + drop(f); + let _ = std::fs::remove_file(&path); + outcome = Err(fail(&path, e)); + last = None; + } else { + outcome = Ok(path); + } + break 'names; + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(e) => { + last = Some(e); + continue 'names; + } + } + } + // A name this folder won't take (too long, a character it + // refuses): the next name. + Err(e) => { + last = Some(e); + continue 'names; + } + } + } + } + let _ = std::fs::remove_file(&aside); + match (outcome, last) { + (Err(_), Some(e)) => Err(fail(folder, e)), + (outcome, _) => outcome, + } +} + +/// `wanderbooks download --book ID ATTACHMENT --to FOLDER`: save a copy of +/// a stored file (one too large for books.get_receipt to return). +pub fn download( + prog: &str, + args: impl IntoIterator, +) -> Result<(), Box> { + let usage = || { + eprintln!("usage: {prog} --book ID [--token NAME] ATTACHMENT --to FOLDER"); + eprintln!(" Save a copy of the stored file ATTACHMENT in FOLDER (a new file)."); + }; + let Some(a) = file_args(args, &usage)? else { + return Ok(()); + }; + let ([attachment], Some(to)) = (a.words.as_slice(), a.to.clone()) else { + usage(); + return Err(exit(2)); + }; + let Some(attachment) = attachment + .to_str() + .filter(|a| a.parse::().is_ok_and(|n| n.to_string() == *a)) + else { + eprintln!("error: an attachment's id is a number"); + return Err(exit(2)); + }; + let fail = |why: String| -> Box { + eprintln!("error: {why}"); + exit(1) + }; + if !to.is_dir() { + return Err(fail(format!("{} isn't a folder", to.display()))); + } + let (status, headers, body) = on_the_socket(&a.common, &a.book, &a.token, |http, base| { + http.get(format!("{base}/a/{attachment}")) + }) + .map_err(fail)?; + let path = keep(status, &headers, &body, attachment, &to).map_err(fail)?; + println!("Saved {} ({} bytes).", path.display(), body.len()); + Ok(()) +} + +/// What `download` does with the server's answer: the server's words if it +/// refused; nothing if the bytes aren't the stored ones; else a new file +/// in `to` (its path). +fn keep( + status: u16, + headers: &reqwest::header::HeaderMap, + body: &[u8], + attachment: &str, + to: &std::path::Path, +) -> Result { + if status != 200 { + return Err(String::from_utf8_lossy(body).to_string()); + } + if !arrived_whole(headers, body) { + return Err( + "the file arrived damaged (its SHA-256 isn't the stored one): nothing was saved".into(), + ); + } + let fallback = format!("attachment-{attachment}"); + let named = saved_name(headers).unwrap_or_else(|| fallback.clone()); + save_new(to, &named, &fallback, body) +} + #[cfg(test)] mod tests { use super::*; + fn disposition(value: &str) -> reqwest::header::HeaderMap { + let mut h = reqwest::header::HeaderMap::new(); + h.insert( + reqwest::header::CONTENT_DISPOSITION, + reqwest::header::HeaderValue::from_str(value).unwrap(), + ); + h + } + + #[test] + fn a_download_is_saved_under_a_plain_visible_name() { + let named = |v: &str| saved_name(&disposition(v)); + // The UTF-8 name first, else the plain one. + assert_eq!( + named("attachment; filename=\"R_sum_.pdf\"; filename*=UTF-8''R%C3%A9sum%C3%A9.pdf") + .as_deref(), + Some("Résumé.pdf") + ); + assert_eq!( + named("attachment; filename=\"a b.pdf\"").as_deref(), + Some("a b.pdf") + ); + // A parameter inside the quoted name is the name, not a parameter. + assert_eq!( + named("attachment; filename=\"x; filename*=UTF-8''.bashrc; y\"").as_deref(), + Some("x; filename*=UTF-8''.bashrc; y") + ); + // No folder, nothing hidden, nothing invisible. + for (given, saved) in [ + ( + "filename*=UTF-8''..%2F..%2F.ssh%2Fauthorized_keys", + Some("_.._.ssh_authorized_keys"), + ), + ("filename*=UTF-8''%2Fetc%2Fpasswd", Some("_etc_passwd")), + ("filename*=UTF-8''.bashrc", Some("bashrc")), + ("filename*=UTF-8''rcpt%E2%80%AEfdp.exe", Some("rcptfdp.exe")), + ("filename*=UTF-8''a%E2%80%8Bb.pdf", Some("ab.pdf")), + ("filename*=UTF-8''a%0Ab.pdf", Some("a_b.pdf")), + ("filename*=UTF-8''..", None), + ("filename*=UTF-8''%20.", None), + ("filename*=UTF-8''%ZZ", None), + ("filename=\"\"", None), + ] { + assert_eq!( + named(&format!("attachment; {given}")).as_deref(), + saved, + "{given}" + ); + } + assert_eq!(saved_name(&reqwest::header::HeaderMap::new()), None); + // Too long: cut on a character, its extension kept. + let long = "收".repeat(90) + ".pdf"; + let cut = plain_name(&long).unwrap(); + assert!( + cut.len() <= SAVED_NAME_MAX && cut.ends_with("收.pdf"), + "{cut}" + ); + assert_eq!( + plain_name(&"a".repeat(SAVED_NAME_MAX)).unwrap().len(), + SAVED_NAME_MAX + ); + assert_eq!(percent_decoded("%E2%9C%93").as_deref(), Some("✓")); + assert_eq!(percent_decoded("%E2%9C"), None, "not UTF-8"); + assert_eq!(percent_decoded("%4"), None); + } + + #[test] + fn only_bytes_with_the_digest_sent_arrived_whole() { + let body = b"%PDF-1.4 stored"; + let digest = base64::Engine::encode( + &base64::engine::general_purpose::STANDARD, + ::digest(body), + ); + let with = |v: &str| { + let mut h = reqwest::header::HeaderMap::new(); + h.insert( + ledger_mcp::local::DIGEST_HEADER, + reqwest::header::HeaderValue::from_str(v).unwrap(), + ); + h + }; + assert!(arrived_whole(&with(&format!("sha-256=:{digest}:")), body)); + assert!(!arrived_whole( + &with(&format!("sha-256=:{digest}:")), + b"%PDF-1.4 changed" + )); + assert!(!arrived_whole(&with(&format!("sha-512=:{digest}:")), body)); + assert!(!arrived_whole(&with(&format!("sha-256={digest}")), body)); + assert!(!arrived_whole(&with("sha-256=:not base64:"), body)); + assert!(!arrived_whole(&reqwest::header::HeaderMap::new(), body)); + } + + #[test] + fn only_a_whole_file_the_server_gave_is_kept() { + let dir = tempfile::tempdir().unwrap(); + let body = b"%PDF-1.4 stored"; + let mut h = disposition("attachment; filename=\"r.pdf\""); + let digest = |b: &[u8]| { + format!( + "sha-256=:{}:", + base64::Engine::encode( + &base64::engine::general_purpose::STANDARD, + ::digest(b) + ) + ) + }; + h.insert( + ledger_mcp::local::DIGEST_HEADER, + reqwest::header::HeaderValue::from_str(&digest(b"something else")).unwrap(), + ); + let e = keep(200, &h, body, "7", dir.path()).unwrap_err(); + assert!(e.contains("arrived damaged"), "{e}"); + let e = keep(404, &h, b"there is no such attachment", "7", dir.path()).unwrap_err(); + assert_eq!(e, "there is no such attachment"); + assert_eq!( + std::fs::read_dir(dir.path()).unwrap().count(), + 0, + "nothing saved" + ); + h.insert( + ledger_mcp::local::DIGEST_HEADER, + reqwest::header::HeaderValue::from_str(&digest(body)).unwrap(), + ); + assert_eq!( + keep(200, &h, body, "7", dir.path()).unwrap(), + dir.path().join("r.pdf") + ); + let bare = reqwest::header::HeaderMap::new(); + assert!( + keep(200, &bare, body, "7", dir.path()).is_err(), + "no digest, no file" + ); + } + + #[test] + fn a_saved_copy_is_new_and_whole_beside_what_is_there() { + let dir = tempfile::tempdir().unwrap(); + let d = dir.path(); + std::fs::write(d.join("r.pdf"), b"theirs").unwrap(); + let first = save_new(d, "r.pdf", "attachment-7", b"ours").unwrap(); + assert_eq!(first, d.join("r (1).pdf")); + let second = save_new(d, "r.pdf", "attachment-7", b"ours").unwrap(); + assert_eq!(second, d.join("r (2).pdf")); + assert_eq!(std::fs::read(d.join("r.pdf")).unwrap(), b"theirs"); + assert_eq!(std::fs::read(&first).unwrap(), b"ours"); + // A name the folder won't take: the fallback. + let long = "x".repeat(300); + assert_eq!( + save_new(d, &long, "attachment-7", b"ours").unwrap(), + d.join("attachment-7") + ); + // Nothing aside is left. + let mut names: Vec = std::fs::read_dir(d) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + names.sort(); + assert_eq!(names, ["attachment-7", "r (1).pdf", "r (2).pdf", "r.pdf"]); + // A folder that can't be written: refused, nothing made. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let ro = d.join("ro"); + std::fs::create_dir(&ro).unwrap(); + std::fs::set_permissions(&ro, std::fs::Permissions::from_mode(0o500)).unwrap(); + if std::fs::write(ro.join("probe"), b"").is_err() { + assert!(save_new(&ro, "r.pdf", "attachment-7", b"ours").is_err()); + } + std::fs::set_permissions(&ro, std::fs::Permissions::from_mode(0o700)).unwrap(); + } + } + #[test] fn close_reads_the_books_state_for_what_it_is() { let leases = Ok(Some(HostMsg::Leases { diff --git a/crates/ledger-host/src/tokens.rs b/crates/ledger-host/src/tokens.rs index f6a60f3..0424876 100644 --- a/crates/ledger-host/src/tokens.rs +++ b/crates/ledger-host/src/tokens.rs @@ -27,7 +27,7 @@ use std::path::{Path, PathBuf}; /// The keyring service tokens are kept under (INVARIANT 1). pub const SERVICE: &str = "wanderbooks"; /// A token's name when none is given. -pub const DEFAULT_NAME: &str = "agent"; +pub const DEFAULT_NAME: &str = ledger_store::auth::DEFAULT_NAME; /// A place tokens are kept. pub trait Keeper: Send + Sync { diff --git a/crates/ledger-mcp/Cargo.toml b/crates/ledger-mcp/Cargo.toml index 1a4d081..4d88061 100644 --- a/crates/ledger-mcp/Cargo.toml +++ b/crates/ledger-mcp/Cargo.toml @@ -20,6 +20,8 @@ ledger-import = { path = "../ledger-import" } ledger-config = { path = "../ledger-config" } ledger-vault = { path = "../ledger-vault" } base64 = "0.22" +# A download's Repr-Digest (local.rs). +sha2 = "0.10" mime_guess = "2" rust_decimal = "1" rmcp = { version = "3", features = ["server", "transport-io", "schemars", "transport-streamable-http-server"] } @@ -32,7 +34,7 @@ serde_json = "1" chrono = "0.4" thiserror = "2" tokio = { version = "1", features = ["rt-multi-thread", "macros", "io-util", "io-std", "sync", "net", "signal", "time"] } -tokio-util = { version = "0.7", features = ["rt"] } +tokio-util = { version = "0.7", features = ["rt", "io", "io-util"] } # A call that panics is recorded before the panic goes on (access.rs). futures-util = "0.3" zip = "9.0.1" @@ -41,8 +43,8 @@ toml = "1" # from memory when dropped, as the vault's own are. zeroize = "1" -# O_NOFOLLOW and O_NONBLOCK for books.import_local_file's open (local_files.rs -# INVARIANT 6): std has no names for these open flags. +# The socket's peer credentials (SO_PEERCRED, getpeereid) for local.rs's +# same-user check: std has no call for them. [target.'cfg(unix)'.dependencies] libc = "0.2" diff --git a/crates/ledger-mcp/handbook/1099.md b/crates/ledger-mcp/handbook/1099.md index c4cf5fc..4e67e68 100644 --- a/crates/ledger-mcp/handbook/1099.md +++ b/crates/ledger-mcp/handbook/1099.md @@ -1,5 +1,5 @@ # 1099s - Mark contractors and landlords with books.set_vendor_1099 (and whether their W-9 is on file), and tag every payment to them with its vendor or party (books.draft_entry vendor or party, books.set_entry_vendor, or books.set_entry_party; books.pay_bill tags automatically). -- books.vendor_1099 is the year-end worksheet for the CPA. With include_tins: true (only when the human asks for the CPA's copy) it carries the full TINs from the vault: hand it over (save_csv_to, a new file outside the books and the import folders) and never repeat those numbers anywhere else. +- books.vendor_1099 is the year-end worksheet for the CPA. With include_tins: true (only when the human asks for the CPA's copy) it carries the full TINs from the vault: hand it over (save_csv_to, a new file outside the books) and never repeat those numbers anywhere else. - Card payments are reported by the card processor (1099-K) and are left out. diff --git a/crates/ledger-mcp/handbook/cpa-handoff.md b/crates/ledger-mcp/handbook/cpa-handoff.md index df17ef1..1e51633 100644 --- a/crates/ledger-mcp/handbook/cpa-handoff.md +++ b/crates/ledger-mcp/handbook/cpa-handoff.md @@ -1,5 +1,5 @@ # CPA handoff -- books.export_cpa_package carries every member's and 1099 payee's full TIN from the vault (the K-1 worksheet and the 1099 worksheet), so it needs save_to: a new .zip or a folder outside the books and the import folders. +- books.export_cpa_package carries every member's and 1099 payee's full TIN from the vault (the K-1 worksheet and the 1099 worksheet), so it needs save_to: a new .zip or a folder outside the books. - Tell the human where it was saved; never open, read, attach, or copy the file, and never echo its TINs. - tin_gaps lists each member or payee whose TIN it lacks and why: collect them (books.set_tin, books.set_w9) and export again. diff --git a/crates/ledger-mcp/handbook/files-in-out.md b/crates/ledger-mcp/handbook/files-in-out.md index 6150ad5..bd2d06e 100644 --- a/crates/ledger-mcp/handbook/files-in-out.md +++ b/crates/ledger-mcp/handbook/files-in-out.md @@ -1,6 +1,8 @@ # Files in and out -- Files come in as base64 (books.attach_receipt, books.file_document), or, for a file already on this computer, with books.import_local_file: an absolute path to a regular file in the server's import folders (by default your downloads folder). Save the file there first; files of the books themselves are never read, and a path that is missing or outside the import folders gets one refusal, whatever is there. -- A W-9 given by path (books.set_w9 w9_file_path) is read the same way. -- Files holding full TINs (books.get_w9, books.vendor_1099 with include_tins, books.export_cpa_package with TINs) are never saved in the import folders. No file with a full TIN goes into the books, however it is filed: the server refuses every file it read or wrote with full TINs (the W-9 books.set_w9 read, and each of those outputs, or a file taken out of a CPA package), and any text file holding one: a TIN the vault keeps, written as nine digits or in the usual groups (3-2-4 with hyphens, spaces or dots; 2-7 with a hyphen or a space), or a number printed as an SSN or EIN is (3-2-4 or 2-7 digits joined by hyphens, not part of a longer number). Other numbers pass, so bank exports (routing and account numbers, long references) go in. It can't recognize an altered copy of a scan, or a TIN the vault doesn't keep written any other way, so never file a W-9 or a TIN worksheet yourself. -- Receipts come back out through books.get_receipt. +- A file comes in one of two ways, named in the tool that files it (books.attach_receipt, books.file_document, books.set_w9, books.import_bank_csv, books.import_chart_csv): + - an upload, for any file up to 25 MiB (books served by `wanderbooks mcp`; on another connection only inline): books.begin_upload with the file's sha256 (`sha256sum`), size, type, name and purpose; run the command it returns with the file's path; then give its upload_id. Its bytes never pass through you. An upload lasts 15 minutes, is used once, and only by the token that began it. + - inline, for a small file (at most 256 KiB): content_base64 with the file's sha256, so a byte lost on the way is caught. +- If begin_upload says the bytes are stored already (existing_attachment_id), link that attachment instead of filing a copy. +- Files holding full TINs (books.get_w9, books.vendor_1099 with include_tins, books.export_cpa_package with TINs) are never saved inside the books. No file with a full TIN goes into the books, however it is filed: the server refuses every file it read or wrote with full TINs (the W-9 books.set_w9 took, and each of those outputs, or a file taken out of a CPA package), and any text file holding one: a TIN the vault keeps, written as nine digits or in the usual groups (3-2-4 with hyphens, spaces or dots; 2-7 with a hyphen or a space), or a number printed as an SSN or EIN is (3-2-4 or 2-7 digits joined by hyphens, not part of a longer number). Other numbers pass, so bank exports (routing and account numbers, long references) go in. It can't recognize an altered copy of a scan, or a TIN the vault doesn't keep written any other way, so never file a W-9 or a TIN worksheet yourself. +- Receipts and documents come back out through books.get_receipt: view bytes gives a file of at most 256 KiB inline; a larger one gives the command (wanderbooks download) that saves a copy in a folder you name (only for books served by `wanderbooks mcp`). diff --git a/crates/ledger-mcp/handbook/parties-and-tins.md b/crates/ledger-mcp/handbook/parties-and-tins.md index 92a317b..63e74aa 100644 --- a/crates/ledger-mcp/handbook/parties-and-tins.md +++ b/crates/ledger-mcp/handbook/parties-and-tins.md @@ -6,6 +6,6 @@ - The post_entry preview lists each party's resolved share (parties) and refuses to post (party_shares, parties_error) until the shares add up to what the entry moved; show the split to the human before confirming. - Every post_entry preview says where the entry stands (party_policy, party_missing, party_suggestions): under the `[books] parties` setting "suggest" (the default), when party_missing, name the likely party to the human and tag it (or add it) before confirming; under "require", posting is refused (party_required) until the entry names a party or is marked as having none (books.set_entry_party with party null and no_party_reason, or draft_entry no_party_reason: only for an entry with truly no counterparty, like a transfer between the company's own accounts); under "off", don't raise it. - Record W-9 facts (books.set_w9) for contractors and landlords before paying them, and W-4 facts (books.set_w4) for employees before their first payroll; name the employee party on every books.run_payroll line. -- Never put a full TIN or SSN anywhere in the books, only the last four digits: the engine refuses any nine-digit number in a party field. The full TIN goes only to the encrypted TIN vault, and every member's and every 1099 payee's belongs there: books.set_tin (party, tin, tin_type: ssn, ein, itin; any role, so each member's SSN for their K-1) and books.set_w9 with tin and tin_type for a payee's W-9, and the signed form with w9_file_path (never books.file_document or books.attach_receipt: the form shows the TIN). +- Never put a full TIN or SSN anywhere in the books, only the last four digits: the engine refuses any nine-digit number in a party field. The full TIN goes only to the encrypted TIN vault, and every member's and every 1099 payee's belongs there: books.set_tin (party, tin, tin_type: ssn, ein, itin; any role, so each member's SSN for their K-1) and books.set_w9 with tin and tin_type for a payee's W-9, and the signed form with w9_upload_id (books.begin_upload purpose w9), or w9_content_base64 with w9_sha256 for one of at most 256 KiB (never books.file_document or books.attach_receipt: the form shows the TIN). - Never repeat a TIN back to the human, in chat, a description, note, file name, version-history message, or anywhere else; tools show the last four only. books.get_w9 writes the form out for the human (never open or read it yourself); books.purge_tin (preview, then confirmed, with a reason) deletes a TIN no longer needed. - books.parties lists the directory, books.party shows one party with the year's totals, and books.show party opens it in the viewer. Archive (books.update_party archived) parties no longer dealt with; never delete history. diff --git a/crates/ledger-mcp/src/lib.rs b/crates/ledger-mcp/src/lib.rs index 3e73a9b..a89ef53 100644 --- a/crates/ledger-mcp/src/lib.rs +++ b/crates/ledger-mcp/src/lib.rs @@ -3,8 +3,7 @@ //! [`run`] is the whole server; the `ledger-mcp` binary and `wanderbooks //! dev-stdio` (one executable, plan D-22) both call it. //! -//! Usage: `ledger-mcp [--data-dir PATH] [--config PATH] [--books-toml PATH] -//! [--import-root DIR]...` +//! Usage: `ledger-mcp [--data-dir PATH] [--config PATH] [--books-toml PATH]` //! //! The data dir holds `journal.jsonl` (the source of truth), `wanderbooks.db` //! (the disposable SQLite projection), `blobs/` (content-addressed receipts), @@ -12,19 +11,16 @@ //! configured base currency. Company settings (name, fiscal year start, //! receipt threshold) come from the books dir's `wanderbooks.toml`, by //! default the one next to the data dir, and reload whenever it changes -//! (settings.rs). `books.import_local_file` and `books.set_w9`'s -//! `w9_file_path` read only from the import roots: each `--import-root`, -//! or by default the downloads folder (local_files.rs); a root that is -//! `/`, the home folder, or holds the books, the vault or a key is refused -//! at startup. +//! (settings.rs). Files come in only as tool input (inline, at most 256 +//! KiB): uploads need the books server's socket (`wanderbooks mcp`). pub mod access; pub mod admin; mod handbook; #[cfg(unix)] pub mod local; -mod local_files; pub mod opening; +mod paths; mod policy; pub mod scopes; mod server; @@ -35,7 +31,6 @@ mod wire; use ledger_core::Currency; use ledger_store::Store; -use local_files::ImportRoots; use policy::ReceiptPolicy; use rmcp::ServiceExt; use serde::Deserialize; @@ -59,7 +54,7 @@ fn default_data_dir() -> PathBuf { fn print_usage(prog: &str) { eprintln!( - "usage: {prog} [--data-dir PATH] [--config PATH] [--books-toml PATH] [--vault-key-file PATH] [--http ADDR] [--import-root DIR]..." + "usage: {prog} [--data-dir PATH] [--config PATH] [--books-toml PATH] [--vault-key-file PATH] [--http ADDR]" ); eprintln!(" --data-dir directory holding journal.jsonl (default ~/.local/share/wanderbooks)"); eprintln!(" --config JSON config file (default /config.json, optional)"); @@ -72,13 +67,10 @@ fn print_usage(prog: &str) { eprintln!( " --http serve over Streamable HTTP at ADDR (e.g. 127.0.0.1:8080) instead of stdio" ); - eprintln!( - " --import-root a folder books.import_local_file may read from (repeatable; default: the downloads folder)" - ); } /// How [`run`] stops when it can't go on: the reason is already printed -/// on stderr, and the number is the process's exit code (2: a bad argument or a refused import folder; 1: books from a newer wanderbooks). A +/// on stderr, and the number is the process's exit code (2: a bad argument; 1: books from a newer wanderbooks). A /// library never ends the process itself, so the books it opened are /// closed properly on the way out; the binaries turn this into the exit. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -95,8 +87,8 @@ impl std::error::Error for Exit {} /// Run the books server with `args` (the command line after the program /// name). `prog` names the program in the usage text: `ledger-mcp`, or /// `wanderbooks dev-stdio`. It returns when the client disconnects; a bad -/// argument, books from a newer wanderbooks, or a refused import root -/// return [`Exit`] after a plain message. +/// argument or books from a newer wanderbooks return [`Exit`] after a +/// plain message. pub fn run( prog: &str, args: impl IntoIterator, @@ -106,7 +98,6 @@ pub fn run( let mut books_toml: Option = None; let mut vault_key_file: Option = None; let mut http_addr: Option = None; - let mut import_roots: Vec = Vec::new(); let mut args = args.into_iter(); while let Some(arg) = args.next() { match arg.as_str() { @@ -129,11 +120,6 @@ pub fn run( "--http" => { http_addr = Some(args.next().ok_or("--http needs an address")?); } - "--import-root" => { - import_roots.push(PathBuf::from( - args.next().ok_or("--import-root needs a folder")?, - )); - } "--help" | "-h" => { print_usage(prog); return Ok(()); @@ -153,11 +139,6 @@ pub fn run( ); return Err(Box::new(Exit(2))); } - let import_roots = if import_roots.is_empty() { - ImportRoots::defaults() - } else { - ImportRoots::new(import_roots) - }; let config_path = config_path.unwrap_or_else(|| data_dir.join("config.json")); // config.json's base currency matters only to books being created: it // is read then, and never otherwise (books that keep their own @@ -232,36 +213,18 @@ pub fn run( } ); let live = std::sync::Arc::new(std::sync::Mutex::new(live)); - // No import root may be `/`, the home folder, or hold the books, the - // vault or a key (local_files.rs INVARIANT 0): a named one stops the - // server here, a default one is dropped. - let settings_now = live.lock().unwrap_or_else(|e| e.into_inner()).current(); - let import_roots = match server::guard_import_roots( - &store, - &settings_now, - &server::Layout::BooksDir, - import_roots, - ) { - Ok(roots) => roots, - Err(why) => { - eprintln!("ledger-mcp: {why}"); - return Err(Box::new(Exit(2))); - } - }; let rt = tokio::runtime::Builder::new_multi_thread() .enable_all() .build()?; if let Some(addr) = http_addr { eprintln!("ledger-mcp: store verified, serving Streamable HTTP on {addr}"); - rt.block_on(serve_http(addr, store, live, import_roots))?; + rt.block_on(serve_http(addr, store, live))?; return Ok(()); } eprintln!("ledger-mcp: store verified, serving over stdio"); - let server = BooksServer::new(store, ReceiptPolicy::default()) - .with_live_settings(live) - .with_import_roots(import_roots); + let server = BooksServer::new(store, ReceiptPolicy::default()).with_live_settings(live); rt.block_on(async { // The wire answers what JSON-RPC requires that rmcp drops (wire.rs). let (input, output, writer) = wire::spawn(tokio::io::stdin(), tokio::io::stdout()); @@ -277,7 +240,6 @@ async fn serve_http( addr: String, store: Store, live: std::sync::Arc>, - import_roots: ImportRoots, ) -> Result<(), Box> { use rmcp::transport::streamable_http_server::{ StreamableHttpService, session::local::LocalSessionManager, @@ -290,8 +252,7 @@ async fn serve_http( let service = StreamableHttpService::new( move || { let server = server::BooksServer::new_shared(store.clone(), ReceiptPolicy::default()) - .with_live_settings(live.clone()) - .with_import_roots(import_roots.clone()); + .with_live_settings(live.clone()); Ok(server) }, Arc::new(LocalSessionManager::default()), diff --git a/crates/ledger-mcp/src/local.rs b/crates/ledger-mcp/src/local.rs index 903e21d..437ccae 100644 --- a/crates/ledger-mcp/src/local.rs +++ b/crates/ledger-mcp/src/local.rs @@ -43,10 +43,20 @@ //! bindings a new session is refused (an initialize), never a binding //! dropped. Tokens are made, listed and revoked only through //! the admin socket. +//! 9. A file's bytes come in only by `PUT /b/{id}/u/{upload}` and go +//! out only by `GET /b/{id}/a/{attachment}` (plan 4.7), each through +//! the same door as 8 and counted for the close like any request: +//! an upload only from the token that began it, a download only with +//! books:read. Each, refused at the door or not, is recorded in the +//! book's access log as `upload` or `download` with its id, and the +//! bytes (kept, or read) and their record are one task that runs to +//! its end if the client hangs up (a process killed in that task +//! excepted); bytes whose record can't be written aren't kept, and +//! the books stop. Lapsed uploads are swept +//! every SWEEP_EVERY. Nothing comes in once the books stopped. //! ``` use crate::admin::{self, Request, Response}; -use crate::local_files::ImportRoots; use crate::policy::ReceiptPolicy; use crate::server::{BooksServer, Layout, SharedStore}; use crate::settings::Live; @@ -75,10 +85,9 @@ pub const SERVER_LOCK: &str = "server.lock"; /// What the server prints on stdout when it is ready (INVARIANT 5). pub const READY: &str = "ready"; -/// How large a request may be. Receipts still ride inside tool calls as -/// base64 until uploads have their own route (P2 slice 5, which brings -/// this down to 1 MiB). -pub const MAX_BODY: usize = 64 * 1024 * 1024; +/// How large an MCP request may be (plan 3.2): files of any size come +/// through uploads (`/b/{book}/u/{upload}`); inline, at most 256 KiB. +pub const MAX_BODY: usize = 1024 * 1024; /// How long the admin socket waits for a request to arrive. const ADMIN_READ_TIMEOUT: Duration = Duration::from_secs(30); @@ -89,6 +98,9 @@ use ledger_store::root::THE_ROOT; /// didn't open, books that don't exist): a client knows it didn't run. pub const REFUSED_HEADER: &str = "x-wanderbooks-refused"; +/// How often each book's lapsed uploads are swept. +const SWEEP_EVERY: Duration = Duration::from_secs(10); + type Service = StreamableHttpService; /// A book the server holds. @@ -103,6 +115,8 @@ enum Book { /// Its access log, where requests refused at the door are /// recorded too (access.rs). access: Arc, + /// Its files on their way in (`/b/{id}/u/{upload}`). + uploads: ledger_store::uploads::Uploads, }, /// Not opened, with why (INVARIANT 3). Failed(String), @@ -121,7 +135,6 @@ impl Book { /// Every book the server holds, and what serving them needs. struct Books { root: Root, - import_roots: ImportRoots, held: RwLock>, /// Ends every MCP session (INVARIANT 6, last). sessions: CancellationToken, @@ -250,17 +263,14 @@ impl Books { let mut live = Live::own(); live.sync(store.ledger().settings(), store.base_currency().code()); let layout = Layout::Root(self.root.path().to_path_buf()); - let roots = crate::server::guard_import_roots( - &store, - &live.current(), - &layout, - self.import_roots.clone(), - ) - .map_err(|e| self.hide(e))?; let access = Arc::new( AccessLog::open(store.data_dir()) .map_err(|e| format!("its access log can't be used: {e}"))?, ); + let uploads = + ledger_store::uploads::Uploads::open(store.data_dir(), chrono::Utc::now().timestamp()) + .map_err(|e| self.hide(e.to_string()))?; + let server_uploads = uploads.clone(); let live = Arc::new(Mutex::new(live)); let store = SharedStore::new(store); let log = access.clone(); @@ -276,9 +286,9 @@ impl Books { Ok( BooksServer::new_shared(store.clone(), ReceiptPolicy::default()) .with_live_settings(live.clone()) - .with_import_roots(roots.clone()) .with_layout(layout.clone()) - .with_access(log.clone()), + .with_access(log.clone()) + .with_uploads(server_uploads.clone()), ) }, Arc::new(sessions()), @@ -288,6 +298,7 @@ impl Books { service, store: shared, access, + uploads, }) } @@ -848,12 +859,29 @@ const SESSION_HEADER: &str = "mcp-session-id"; /// live sessions). const MAX_SESSIONS: usize = 100_000; +/// What a request asks for, as the access log names it: an MCP request +/// (action `connect`, its method), an upload (`upload`, its id) or a +/// download (`download`, the attachment's id). +struct Asked { + action: &'static str, + subject: String, +} + +impl Asked { + fn connect(method: &axum::http::Method) -> Self { + Self { + action: "connect", + subject: method.as_str().to_string(), + } + } +} + /// A request refused at the door: recorded in the book's access log /// (access.rs INVARIANT 3), then answered with `status` and `why`. async fn refused_at_door( book: &Book, principal: Principal, - method: &axum::http::Method, + asked: &Asked, code: &'static str, status: axum::http::StatusCode, why: String, @@ -865,8 +893,9 @@ async fn refused_at_door( principal, client: None, request_id: None, - action: "connect".into(), - subject: method.as_str().to_string(), + action: asked.action.into(), + // Masked, then cut, by the log itself (access.rs INVARIANT 2). + subject: asked.subject.clone(), args_digest: None, result: Outcome::Refused(code.into()), head_after: None, @@ -892,34 +921,31 @@ async fn refused_at_door( response } -/// `/b/{book}/mcp`: the book's MCP service, for a request carrying a token -/// for that book (INVARIANT 8). -async fn mcp( - axum::extract::State(books): axum::extract::State>, - axum::extract::Path(book): axum::extract::Path, - mut request: axum::extract::Request, -) -> axum::response::Response { +/// The door to `book` (INVARIANT 8): the books held open, the server not +/// closing, and a live token for those books; else the answer refusing +/// the request (recorded in the books' access log, where there is one). +async fn door( + books: &Arc, + book: &str, + asked: &Asked, + headers: &axum::http::HeaderMap, +) -> Result<(Book, Token), Box> { use axum::http::StatusCode; use axum::response::IntoResponse; - use tower::ServiceExt; - // Counted before the closing check, so closing never misses it. - let counted = (request.method() != axum::http::Method::GET).then(|| InFlight::new(&books)); - books - .last_arrival - .store(books.started.elapsed().as_millis() as u64, Ordering::SeqCst); let closing = books.closing.is_cancelled(); - let found = books.held().get(&book).cloned(); - let (service, held) = match found { + let found = books.held().get(book).cloned(); + let held = match found { Some(Book::Failed(why)) if !closing => { - return ( + return Err(( StatusCode::SERVICE_UNAVAILABLE, [(REFUSED_HEADER, "not-open")], format!("the books {book} didn't open: {why}"), ) - .into_response(); + .into_response() + .into()); } None if !closing => { - return ( + return Err(( StatusCode::NOT_FOUND, [(REFUSED_HEADER, "no-such-books")], format!( @@ -927,22 +953,20 @@ async fn mcp( book.chars().take(64).collect::() ), ) - .into_response(); + .into_response() + .into()); } - Some(b @ Book::Open { .. }) => match &b { - Book::Open { service, .. } => (service.clone(), b.clone()), - Book::Failed(_) => unreachable!(), - }, + Some(b @ Book::Open { .. }) => b, _ => { - return ( + return Err(( StatusCode::SERVICE_UNAVAILABLE, "the books are closing: this request wasn't run", ) - .into_response(); + .into_response() + .into()); } }; - let method = request.method().clone(); - let token = books.bearer(request.headers()); + let token = books.bearer(headers); let principal = |t: &Token| Principal { id: t.id.clone(), kind: "agent".into(), @@ -952,41 +976,382 @@ async fn mcp( let who = token .as_ref() .map_or_else(|_| Principal::unknown(), principal); - return refused_at_door( + return Err(refused_at_door( &held, who, - &method, + asked, crate::access::CLOSING, StatusCode::SERVICE_UNAVAILABLE, "the books are closing: this request wasn't run".into(), ) - .await; + .await + .into()); } let token = match token { Ok(t) => t, Err((code, why)) => { - return refused_at_door( + return Err(refused_at_door( &held, Principal::unknown(), - &method, + asked, code, StatusCode::UNAUTHORIZED, why, ) - .await; + .await + .into()); } }; if token.book != book { - return refused_at_door( + return Err(refused_at_door( &held, principal(&token), - &method, + asked, "wrong_books", StatusCode::FORBIDDEN, format!("this token is for the books {}, not {book}", token.book), ) - .await; + .await + .into()); } + Ok((held, token)) +} + +/// `PUT /b/{book}/u/{upload}`: the bytes of an upload begun with +/// `books.begin_upload`, from the token that began it (INVARIANT 9; plan +/// 4.7): at most the upload's limit, kept only if exactly the size and +/// SHA-256 declared, once; recorded in the access log (action `upload`). +async fn upload( + axum::extract::State(books): axum::extract::State>, + axum::extract::Path((book, id)): axum::extract::Path<(String, String)>, + request: axum::extract::Request, +) -> axum::response::Response { + use axum::http::StatusCode; + use axum::response::IntoResponse; + use ledger_store::uploads::UploadError; + let _counted = InFlight::new(&books); + books + .last_arrival + .store(books.started.elapsed().as_millis() as u64, Ordering::SeqCst); + let asked = Asked { + action: "upload", + subject: id.clone(), + }; + let (held, token) = match door(&books, &book, &asked, request.headers()).await { + Ok(passed) => passed, + Err(refused) => { + // What was sent is read and dropped (up to the most an upload + // can be: only this account reaches the socket), so the client + // gets the refusal, not a connection cut mid-send. + use futures_util::StreamExt; + let mut rest = request.into_body().into_data_stream(); + let mut read = 0u64; + while let Some(Ok(chunk)) = rest.next().await { + read += chunk.len() as u64; + if read > ledger_store::uploads::MAX_UPLOAD { + break; + } + } + return *refused; + } + }; + let Book::Open { + access, + uploads, + store, + .. + } = &held + else { + unreachable!("the door lets only open books through") + }; + // Streamed to the disk as it comes, never held whole; `receive` reads + // no more than the size declared and one byte. The bytes and their + // record are one blocking task, which runs to its end even if the + // client hangs up: never bytes kept without their record, and a close + // waits for both (the count goes with them). + use futures_util::TryStreamExt; + let stream = request + .into_body() + .into_data_stream() + .map_err(std::io::Error::other); + let mut body = tokio_util::io::SyncIoBridge::new(tokio_util::io::StreamReader::new(stream)); + let (uploads, access, stopped) = (uploads.clone(), access.clone(), store.is_stopped()); + let task = tokio::task::spawn_blocking(move || { + let _counted = _counted; + // The books server stopped after an internal error: nothing more + // comes in (SharedStore INVARIANT 2). + let received = if stopped { + Err(UploadError::Bad(crate::server::STOPPED.into())) + } else { + uploads.receive(&id, &token.id, &mut body, chrono::Utc::now().timestamp()) + }; + // Refused: the rest of what was sent (up to the most an upload + // can be) is read and dropped, so the client gets the answer, not + // a connection cut while it was still sending. + if received.is_err() { + use std::io::Read; + let rest = ledger_store::uploads::MAX_UPLOAD + 1; + let _ = std::io::copy(&mut (&mut body).take(rest), &mut std::io::sink()); + } + let (status, outcome, text) = match &received { + Ok(g) => ( + StatusCode::OK, + Outcome::Ok, + serde_json::json!({"received": g.id, "size": g.size, "sha256": g.sha256}) + .to_string(), + ), + Err(e) => { + let (status, code) = match e { + _ if stopped => (StatusCode::SERVICE_UNAVAILABLE, "stopped"), + UploadError::Unknown(_) => (StatusCode::NOT_FOUND, "no_such_upload"), + UploadError::NotYours => (StatusCode::FORBIDDEN, "not_yours"), + UploadError::AlreadySent => (StatusCode::CONFLICT, "already_sent"), + UploadError::InUse => (StatusCode::CONFLICT, "in_use"), + UploadError::WrongSize { .. } + | UploadError::TooMuch { .. } + | UploadError::WrongHash => (StatusCode::BAD_REQUEST, "not_as_declared"), + _ => (StatusCode::INTERNAL_SERVER_ERROR, "failed"), + }; + (status, Outcome::Refused(code.into()), e.to_string()) + } + }; + let record = Access { + principal: Principal { + id: token.id.clone(), + kind: "agent".into(), + label: token.name.clone(), + }, + client: None, + request_id: None, + action: "upload".into(), + subject: id.clone(), + args_digest: None, + result: outcome.clone(), + head_after: None, + }; + let recorded = access.record(&record).is_ok(); + if !recorded && received.is_ok() { + // Never bytes kept without their record: sent again later. + uploads.discard(&id); + } + (status, outcome, text, recorded) + }); + let Ok((status, outcome, text, true)) = task.await else { + store.stop(); + return ( + StatusCode::SERVICE_UNAVAILABLE, + [(REFUSED_HEADER, "not-recorded")], + crate::access::NOT_RECORDED, + ) + .into_response(); + }; + match outcome { + Outcome::Refused(code) => (status, [(REFUSED_HEADER, code)], text).into_response(), + _ => (status, text).into_response(), + } +} + +/// `GET /b/{book}/a/{attachment}`: a stored file's bytes, for a token +/// with books:read (INVARIANT 9; what `wanderbooks download` asks for); +/// recorded in the access log (action `download`). +async fn download( + axum::extract::State(books): axum::extract::State>, + axum::extract::Path((book, attachment)): axum::extract::Path<(String, String)>, + request: axum::extract::Request, +) -> axum::response::Response { + use axum::http::StatusCode; + use axum::response::IntoResponse; + // GET only (axum hands a HEAD to a GET route): a HEAD would read the + // file and record a download that sent nothing. + if request.method() != axum::http::Method::GET { + return ( + StatusCode::METHOD_NOT_ALLOWED, + [(axum::http::header::ALLOW, "GET")], + "a download is a GET", + ) + .into_response(); + } + let _counted = InFlight::new(&books); + books + .last_arrival + .store(books.started.elapsed().as_millis() as u64, Ordering::SeqCst); + let asked = Asked { + action: "download", + subject: attachment.clone(), + }; + let (held, token) = match door(&books, &book, &asked, request.headers()).await { + Ok(passed) => passed, + Err(refused) => return *refused, + }; + let Book::Open { access, store, .. } = &held else { + unreachable!("the door lets only open books through") + }; + // The read and its record are one blocking task, run to its end even + // if the client hangs up, and counted until then. + let (access, store_for) = (access.clone(), store.clone()); + let task = tokio::task::spawn_blocking(move || { + let _counted = _counted; + // An id as the books write it (no sign, no leading zeros), so the + // log holds no alias of it. + let id = attachment + .parse::() + .ok() + .filter(|n| n.to_string() == attachment); + let found = match (token.scopes.contains(&Scope::Read), id) { + // Stopped after an internal error: as for uploads. + _ if store_for.is_stopped() => Err(( + StatusCode::SERVICE_UNAVAILABLE, + "stopped", + crate::server::STOPPED.to_string(), + )), + (false, _) => Err(( + StatusCode::FORBIDDEN, + "insufficient_scope", + format!( + "this agent's token can't do that: a download needs books:read, and its token ({:?}) hasn't it", + token.name + ), + )), + (true, None) => Err(( + StatusCode::NOT_FOUND, + "no_such_attachment", + "there is no such attachment".to_string(), + )), + (true, Some(id)) => match store_for.attachment_file(id) { + Ok(Some(file)) => Ok(file), + Ok(None) => Err(( + StatusCode::NOT_FOUND, + "no_such_attachment", + "there is no such attachment".to_string(), + )), + // Stopped, or the stored file damaged: not a missing one. + Err(why) => Err((StatusCode::INTERNAL_SERVER_ERROR, "failed", why)), + }, + }; + let record = Access { + principal: Principal { + id: token.id.clone(), + kind: "agent".into(), + label: token.name.clone(), + }, + client: None, + request_id: None, + action: "download".into(), + subject: attachment.clone(), + args_digest: None, + result: match &found { + Ok(_) => Outcome::Ok, + Err((_, code, _)) => Outcome::Refused((*code).into()), + }, + head_after: None, + }; + let recorded = access.record(&record).is_ok(); + (found, recorded) + }); + let Ok((found, true)) = task.await else { + store.stop(); + return ( + StatusCode::SERVICE_UNAVAILABLE, + [(REFUSED_HEADER, "not-recorded")], + crate::access::NOT_RECORDED, + ) + .into_response(); + }; + match found { + Ok((name, mime, bytes)) => { + let digest = format!( + "sha-256=:{}:", + base64::Engine::encode( + &base64::engine::general_purpose::STANDARD, + ::digest(&bytes) + ) + ); + let mime = axum::http::HeaderValue::from_str(&mime).unwrap_or( + axum::http::HeaderValue::from_static("application/octet-stream"), + ); + ( + StatusCode::OK, + [ + (axum::http::header::CONTENT_TYPE, mime), + ( + axum::http::header::CONTENT_DISPOSITION, + axum::http::HeaderValue::from_str(&disposition(&name)) + .expect("only visible ASCII"), + ), + ( + axum::http::header::X_CONTENT_TYPE_OPTIONS, + axum::http::HeaderValue::from_static("nosniff"), + ), + ( + axum::http::HeaderName::from_static(DIGEST_HEADER), + axum::http::HeaderValue::from_str(&digest).expect("base64 is ASCII"), + ), + ], + bytes, + ) + .into_response() + } + Err((status, code, why)) => (status, [(REFUSED_HEADER, code)], why).into_response(), + } +} + +/// The header a download's SHA-256 rides in (RFC 9530 Repr-Digest), so +/// `wanderbooks download` keeps only what was stored. +pub const DIGEST_HEADER: &str = "repr-digest"; + +/// A download's Content-Disposition (RFC 6266): its name in ASCII, every +/// other character `_`, and whole in UTF-8 as `filename*` (RFC 8187). +fn disposition(name: &str) -> String { + let plain: String = name + .chars() + .map(|c| { + if (c.is_ascii_graphic() || c == ' ') && c != '"' && c != '\\' { + c + } else { + '_' + } + }) + .collect(); + let mut encoded = String::new(); + for b in name.bytes() { + if b.is_ascii_alphanumeric() || b"!#$&+-.^_`|~".contains(&b) { + encoded.push(b as char); + } else { + encoded.push_str(&format!("%{b:02X}")); + } + } + format!("attachment; filename=\"{plain}\"; filename*=UTF-8''{encoded}") +} + +/// `/b/{book}/mcp`: the book's MCP service, for a request carrying a token +/// for that book (INVARIANT 8). +async fn mcp( + axum::extract::State(books): axum::extract::State>, + axum::extract::Path(book): axum::extract::Path, + mut request: axum::extract::Request, +) -> axum::response::Response { + use axum::http::StatusCode; + use tower::ServiceExt; + // Counted before the closing check, so closing never misses it. + let counted = (request.method() != axum::http::Method::GET).then(|| InFlight::new(&books)); + books + .last_arrival + .store(books.started.elapsed().as_millis() as u64, Ordering::SeqCst); + let method = request.method().clone(); + let (held, token) = match door(&books, &book, &Asked::connect(&method), request.headers()).await + { + Ok(passed) => passed, + Err(refused) => return *refused, + }; + let Book::Open { service, .. } = &held else { + unreachable!("the door lets only open books through") + }; + let service = service.clone(); + let principal = |t: &Token| Principal { + id: t.id.clone(), + kind: "agent".into(), + label: t.name.clone(), + }; let session = request .headers() .get(SESSION_HEADER) @@ -1024,7 +1389,7 @@ async fn mcp( return refused_at_door( &held, principal(&token), - &method, + &Asked::connect(&method), "too_many_sessions", StatusCode::SERVICE_UNAVAILABLE, "the books server has too many sessions open: end some, then try again".into(), @@ -1035,7 +1400,7 @@ async fn mcp( return refused_at_door( &held, principal(&token), - &method, + &Asked::connect(&method), "session_token", StatusCode::FORBIDDEN, "this session was opened with another token".into(), @@ -1154,7 +1519,7 @@ fn bind(run: &Path, name: &str) -> std::io::Result { } fn print_usage(prog: &str) { - eprintln!("usage: {prog} [--root DIR] [--import-root DIR]..."); + eprintln!("usage: {prog} [--root DIR]"); eprintln!(" The books server for the data root: started by the host, not by hand."); eprintln!(" It runs until its standard input ends, or SIGTERM, SIGINT or SIGHUP."); } @@ -1166,14 +1531,10 @@ pub fn run( args: impl IntoIterator, ) -> Result<(), Box> { let mut root: Option = None; - let mut import_roots: Vec = Vec::new(); let mut args = args.into_iter(); while let Some(arg) = args.next() { match arg.as_str() { "--root" => root = Some(PathBuf::from(args.next().ok_or("--root needs a folder")?)), - "--import-root" => import_roots.push(PathBuf::from( - args.next().ok_or("--import-root needs a folder")?, - )), "--help" | "-h" => { print_usage(prog); return Ok(()); @@ -1240,14 +1601,8 @@ pub fn run( std::fs::set_permissions(&lock_path, std::fs::Permissions::from_mode(0o600)) .map_err(|e| format!("can't make the server lock private: {}", e.kind()))?; } - let import_roots = if import_roots.is_empty() { - ImportRoots::defaults() - } else { - ImportRoots::new(import_roots) - }; let books = Arc::new(Books { root: root.clone(), - import_roots, held: RwLock::new(BTreeMap::new()), sessions: CancellationToken::new(), closing: CancellationToken::new(), @@ -1326,9 +1681,37 @@ async fn serve( } let held: Vec = books.held().keys().cloned().collect(); eprintln!("wanderbooks server: ready, holding {} books", held.len()); + // Every open book's lapsed uploads go within SWEEP_EVERY of lapsing, + // whatever else the server is doing (uploads.rs INVARIANT 3). + let (sweeping, sweep_stop) = (books.clone(), stop.clone()); + tokio::spawn(async move { + let mut tick = tokio::time::interval(SWEEP_EVERY); + loop { + tokio::select! { + () = sweep_stop.cancelled() => break, + _ = tick.tick() => { + let uploads: Vec<_> = sweeping + .held() + .values() + .filter_map(|b| match b { + Book::Open { uploads, .. } => Some(uploads.clone()), + Book::Failed(_) => None, + }) + .collect(); + let now = chrono::Utc::now().timestamp(); + let _ = tokio::task::spawn_blocking(move || { + uploads.iter().for_each(|u| u.sweep(now)); + }) + .await; + } + } + } + }); let app = axum::Router::new() .route("/b/{book}/mcp", axum::routing::any(mcp)) + .route("/b/{book}/u/{upload}", axum::routing::put(upload)) + .route("/b/{book}/a/{attachment}", axum::routing::get(download)) .fallback(not_found) .with_state(books.clone()); // The HTTP side keeps answering (refusing, once closing) until the diff --git a/crates/ledger-mcp/src/local_files.rs b/crates/ledger-mcp/src/local_files.rs deleted file mode 100644 index fa90f15..0000000 --- a/crates/ledger-mcp/src/local_files.rs +++ /dev/null @@ -1,1430 +0,0 @@ -//! Files the server reads from its own disk for an agent -//! (`books.import_local_file`, `books.set_w9`'s `w9_file_path`): only -//! regular files inside the import roots, and never a file of the books -//! themselves. -//! -//! ```text -//! GOAL: an agent can file a receipt it saved (a download, a rendered PDF, -//! a pasted screenshot) without being able to read the books' own -//! files, the TIN vault's key, or anything else of the human's, -//! through the server. -//! SOURCE: DECISION (owner, 2026-10-16; plan/served.md section 6, -//! "Immediate hole (P0)"): an explicit --import-root allowlist -//! (default: the downloads folder only), canonicalized, refusing -//! the data dir, the books dir, the vault dir, the key's dir, -//! symlink escapes, and non-regular files; the P0 reviews -//! (2026-10-16) for INVARIANTS 0, 2, 3 and 6. -//! INVARIANTS: -//! 0. No import root is `/`, the home folder or a folder holding it, a -//! folder holding the books dir, the data dir, the vault or a key -//! file, or a folder inside the books dir, the data dir, the vault or -//! ~/.config/wanderbooks (ImportRoots::guard): such a root is refused at startup when -//! named (--import-root) and dropped when it is the default. The -//! default is the downloads folder, falling back to ~/Downloads, and -//! only when that is a real folder that isn't the home folder; else -//! there are no roots, and every read says so (NO_ROOTS). -//! 1. A path is read only if it is absolute, has no `.` or `..` name, -//! and where it leads (every symlink resolved) is a regular file -//! strictly inside a canonical import root. -//! 2. A path that is missing, a dangling link, outside every root, or -//! with a `.` or `..` name gets one refusal, NO_SUCH_FILE, worded for -//! all: a refusal never tells what lies outside the roots, or whether -//! a link's target exists. A `.` or `..` name is refused before -//! anything is looked up: resolving `a/b/..` succeeds only when `b` -//! is a folder, which would tell whether folders outside the roots -//! exist. -//! 3. Inside a root, a path in a sealed place (the data dir, the vault, -//! the key files and the folders Protected::of seals) is refused as -//! the books' or the TIN vault's, existing or not; one elsewhere in -//! the books dir is refused with "save a copy first". -//! 4. A file with more than one hard link is refused: a hard link would -//! carry a protected file into a root without a symlink to follow. -//! 5. A refusal names the rule it applies, never a path. -//! 6. The file read is the file checked: it is opened by a walk from -//! `/`, one name of its canonical path at a time, never following a -//! symlink (a folder swapped for a link after the check fails the -//! walk) and never blocking (a pipe swapped in can't hang a thread); -//! no folder on the walk, and not the file, is a protected place (by -//! device and inode: a protected folder renamed into a root); and the -//! file opened is the checked file (device and inode, and size, -//! modification and change times: a filesystem may hand a deleted -//! file's inode number to the next file made), with one link. -//! ``` - -use std::io::Read; -use std::path::{Component, Path, PathBuf}; - -/// The folders `books.import_local_file` may read from. -#[derive(Clone, Debug)] -pub struct ImportRoots { - dirs: Vec, - /// Named with --import-root (a bad one is refused), not the default - /// (a bad one is dropped). - explicit: bool, -} - -/// What is never read through a path, even inside an import root -/// (INVARIANT 3). -#[derive(Clone, Debug, Default)] -pub struct Protected { - /// The books' own records and the TIN vault's: the data dir, the - /// vault, the key files and the folders [`Protected::of`] seals. - pub sealed: Vec, - /// The books dir: its other files (the settings, the company's papers) - /// are filed from a copy, never in place. - pub books: Vec, -} - -/// Where the books keep what an import must never read. -#[derive(Clone, Debug, Default)] -pub struct Places { - /// The data dir (the journal, the database, the receipts). - pub data_dir: PathBuf, - /// The books dir (the data dir's parent). - pub books_dir: PathBuf, - /// The TIN vault's folder. - pub vault_dir: PathBuf, - /// The vault key files: the one in force and the default one. - pub key_files: Vec, - /// The home folder, if known. - pub home: Option, -} - -impl Places { - /// What no import root may hold (INVARIANT 0): the data dir, the books - /// dir, the vault, each key file and where it leads, and the - /// wanderbooks folder under the home folder's .config. - pub fn keep_out(&self) -> Vec { - let mut out = vec![ - self.data_dir.clone(), - self.books_dir.clone(), - self.vault_dir.clone(), - ]; - for k in &self.key_files { - out.push(k.clone()); - out.push(resolved(k)); - } - out.extend(self.home_config()); - out - } - - /// `~/.config/wanderbooks`, whatever XDG_CONFIG_HOME says: a key left - /// there from before still unlocks the vault. - fn home_config(&self) -> Option { - self.home - .as_ref() - .map(|h| h.join(".config").join("wanderbooks")) - } -} - -impl Protected { - /// What these books protect, given the import roots in use. - /// - /// ```text - /// GOAL: the books and the vault key are never read through an import, - /// and sealing them never blocks every import. - /// SOURCE: DECISION (P0 review round 2, 2026-10-16). - /// INVARIANTS: - /// 1. Sealed always: the data dir, the vault, each key file (the one - /// in force and the default) and where it leads, and - /// ~/.config/wanderbooks with where it leads. - /// 2. A key file's folder is sealed too, but only when it is not the - /// home folder, not a root, and not a folder holding a root - /// (sealing it would seal everything an import can read). - /// 3. The books dir is `books` (filed from a copy, INVARIANT 3). - /// ``` - pub fn of(places: &Places, roots: &[PathBuf]) -> Self { - let home = places.home.as_deref().map(resolved); - let mut sealed = vec![ - places.data_dir.clone(), - resolved(&places.data_dir), - places.vault_dir.clone(), - resolved(&places.vault_dir), - ]; - if let Some(c) = places.home_config() { - sealed.push(resolved(&c)); - sealed.push(c); - } - for k in &places.key_files { - for file in [k.clone(), resolved(k)] { - if let Some(folder) = file.parent().map(resolved) { - let holds_a_root = roots.iter().any(|r| r.starts_with(&folder)); - if home.as_ref() != Some(&folder) && !holds_a_root { - sealed.push(folder); - } - } - sealed.push(file); - } - } - Protected { - sealed, - books: vec![places.books_dir.clone()], - } - } - - /// Device and inode of every protected place that exists (links - /// followed): what no step of an open may land on (INVARIANT 6). - #[cfg(unix)] - pub fn inodes(&self) -> Vec<(u64, u64)> { - use std::os::unix::fs::MetadataExt; - self.sealed - .iter() - .chain(&self.books) - .filter_map(|p| std::fs::metadata(p).ok()) - .map(|m| (m.dev(), m.ino())) - .collect() - } - - /// No inodes off Unix: the walk itself is Unix only. - #[cfg(not(unix))] - pub fn inodes(&self) -> Vec<(u64, u64)> { - Vec::new() - } -} - -/// Why `root` can't be an import root, if it can't (INVARIANT 0): it is -/// `/`, the home folder or a folder holding it, a folder holding one of -/// `keep_out`, or a folder inside one. A root that doesn't exist reads -/// nothing, and is let be. -pub fn refused_root( - root: &Path, - home: Option<&Path>, - keep_out: &[PathBuf], -) -> Option<&'static str> { - let root = root.canonicalize().ok()?; - if root.parent().is_none() { - return Some("it is the whole filesystem"); - } - if home.is_some_and(|h| resolved(h).starts_with(&root)) { - return Some("it is the home folder, or holds it"); - } - if keep_out.iter().any(|k| resolved(k).starts_with(&root)) { - return Some("it holds the books, the TIN vault, or a vault key"); - } - // Inside them, every file would be refused (INVARIANT 3), with a - // remedy (save a copy in an import folder) that leads back in. - if keep_out.iter().any(|k| root.starts_with(resolved(k))) { - return Some("it is inside the books or the TIN vault, whose files are never imported"); - } - None -} - -impl ImportRoots { - /// Exactly these folders (`--import-root`, repeatable). - pub fn new(dirs: Vec) -> Self { - Self { - dirs, - explicit: true, - } - } - - /// The default when no `--import-root` is given: the user's downloads - /// folder, only (plan/served.md section 6; INVARIANT 0). - pub fn defaults() -> Self { - Self::defaults_from( - std::env::var_os("XDG_DOWNLOAD_DIR"), - std::env::var_os("HOME").map(PathBuf::from), - ) - } - - /// [`Self::defaults`] for these values of `XDG_DOWNLOAD_DIR` and - /// `HOME`. - pub fn defaults_from(xdg: Option, home: Option) -> Self { - Self { - dirs: default_root(xdg, home.as_deref()).into_iter().collect(), - explicit: false, - } - } - - /// The roots, with every one [`refused_root`] refuses taken out, and - /// those refused with why. A server refuses to start with a named - /// root refused; a refused default is just dropped. - pub fn guard( - self, - home: Option<&Path>, - keep_out: &[PathBuf], - ) -> (Self, Vec<(PathBuf, &'static str)>) { - let mut refused = Vec::new(); - let dirs = self - .dirs - .into_iter() - .filter(|d| match refused_root(d, home, keep_out) { - Some(why) => { - refused.push((d.clone(), why)); - false - } - None => true, - }) - .collect(); - ( - Self { - dirs, - explicit: self.explicit, - }, - refused, - ) - } - - /// Whether the roots were named (`--import-root`), not the default. - pub fn explicit(&self) -> bool { - self.explicit - } - - /// The roots in use: each that exists, canonical, and never `/` or a - /// folder holding the home folder, whatever it has become since the - /// server started (INVARIANT 0). - pub fn usable(&self) -> Vec { - let home = std::env::var_os("HOME").map(PathBuf::from); - self.dirs - .iter() - .filter(|d| refused_root(d, home.as_deref(), &[]).is_none()) - .filter_map(|r| r.canonicalize().ok()) - .filter(|r| r.is_dir()) - .collect() - } - - /// Whether `path` (existing or not) is strictly inside a root: where a - /// file it names could be read back in. - pub fn contains(&self, path: &Path) -> bool { - inside(&self.usable(), &resolved(path)) - } - - /// The bytes of the file at `path`, if the invariants allow it, with - /// its canonical path (for the file name and MIME type). `limit` is - /// the most bytes accepted. - pub fn read( - &self, - path: &str, - protected: &Protected, - limit: usize, - ) -> Result<(PathBuf, Vec), String> { - let roots = self.usable(); - // Where the protected places are now, before anything is checked: - // one moved into a root later is still known by its inode. - let avoid = protected.inodes(); - let (canonical, checked) = inspect(&roots, path, protected)?; - let bytes = read_checked(&canonical, &checked, &avoid, limit)?; - Ok((canonical, bytes)) - } - - /// The canonical path of `path` if it may be read (INVARIANTS 1-5). - #[cfg(test)] - pub fn check(&self, path: &str, protected: &Protected) -> Result { - inspect(&self.usable(), path, protected).map(|(p, _)| p) - } -} - -/// The default import root (INVARIANT 0): `$XDG_DOWNLOAD_DIR` (as -/// user-dirs.dirs writes it, "$HOME/Downloads"), else `~/Downloads`; each -/// only if it is an absolute, real folder that isn't `/`, the home folder -/// or a folder holding it. -fn default_root(xdg: Option, home: Option<&Path>) -> Option { - let ok = |dir: PathBuf| { - (dir.is_absolute() && dir.is_dir() && refused_root(&dir, home, &[]).is_none()) - .then(|| dir.canonicalize().ok()) - .flatten() - }; - let from_xdg = xdg.and_then(|x| { - let x = x.to_string_lossy().into_owned(); - let rest = x - .strip_prefix("${HOME}") - .or_else(|| x.strip_prefix("$HOME")); - match (rest, home) { - (Some(rest), Some(h)) => Some(h.join(rest.trim_start_matches('/'))), - (Some(_), None) => None, - (None, _) => Some(PathBuf::from(x)), - } - }); - from_xdg - .and_then(ok) - .or_else(|| home.and_then(|h| ok(h.join("Downloads")))) -} - -/// Whether `path` has a `.` or `..` name (INVARIANT 2), however it is -/// separated (`/`, or a backslash where that separates names too). -fn has_dot_name(path: &str) -> bool { - path.split(std::path::is_separator) - .any(|name| name == "." || name == "..") -} - -/// Whether `place` is strictly inside one of `roots` (canonical). -fn inside(roots: &[PathBuf], place: &Path) -> bool { - roots.iter().any(|r| place != r && place.starts_with(r)) -} - -/// The canonical path of `path` if it may be read from `roots` -/// (INVARIANTS 1-5), with the checked file's metadata. -fn inspect( - roots: &[PathBuf], - path: &str, - protected: &Protected, -) -> Result<(PathBuf, std::fs::Metadata), String> { - if roots.is_empty() { - return Err(NO_ROOTS.to_string()); - } - let given = Path::new(path); - if !given.is_absolute() { - return Err("path must be absolute".to_string()); - } - // INVARIANT 2: no `.` or `..` name, before anything is looked up. - // Split by hand: `Path::components` drops a `.` that isn't first. - if has_dot_name(path) { - return Err(NO_SUCH_FILE.to_string()); - } - // Where the path leads, existing or not, every link followed, a - // dangling one included (INVARIANT 2): the kernel's own answer when - // there is a file there (nothing there, a link loop, or a name under - // a file, `shot.png/sub/..`, is missing), else where the links lead. - // Outside every root, nothing more is said. - let canonical = given.canonicalize(); - let place = match &canonical { - Ok(c) => c.clone(), - Err(_) => resolved(given), - }; - if !inside(roots, &place) { - return Err(NO_SUCH_FILE.to_string()); - } - let under = |dirs: &[PathBuf]| { - dirs.iter() - .map(|p| resolved(p)) - .any(|p| place.starts_with(&p)) - }; - if under(&protected.sealed) { - return Err(SEALED.to_string()); - } - if under(&protected.books) { - return Err(IN_THE_BOOKS.to_string()); - } - if canonical.is_err() { - return Err(NO_SUCH_FILE.to_string()); - } - let meta = std::fs::metadata(&place).map_err(|e| cant_read(&e))?; - if !meta.is_file() { - return Err(NOT_REGULAR.into()); - } - if hard_linked(&meta) { - return Err(HARD_LINKED.to_string()); - } - Ok((place, meta)) -} - -/// Open and read the file `inspect` checked (INVARIANT 6): by a walk that -/// never follows a symlink, never blocks, and never lands on a protected -/// place, and only if the handle is the checked file. -fn read_checked( - canonical: &Path, - checked: &std::fs::Metadata, - avoid: &[(u64, u64)], - limit: usize, -) -> Result, String> { - let mut file = open_walk(canonical, avoid)?; - // The handle must still be the regular file just checked. - let meta = file.metadata().map_err(|e| cant_read(&e))?; - if !meta.is_file() || hard_linked(&meta) { - return Err(NOT_REGULAR.into()); - } - same_file(checked, &meta)?; - // At most one byte past the limit is read, whatever the file's size - // says (it may grow while it is read). - let mut bytes = Vec::new(); - file.by_ref() - .take(limit as u64 + 1) - .read_to_end(&mut bytes) - .map_err(|e| cant_read(&e))?; - if bytes.len() > limit { - return Err("file exceeds 25 MB limit".to_string()); - } - Ok(bytes) -} - -/// Open `canonical` by walking it from `/` (INVARIANT 6): each folder -/// with `openat(O_DIRECTORY | O_NOFOLLOW)` on the one before, the file -/// with `openat(O_NOFOLLOW | O_NONBLOCK)`. A name on the path swapped for -/// a symlink since the check fails the walk (ELOOP, ENOTDIR), and so does -/// a step onto a protected place (`avoid`: device and inode). -#[cfg(unix)] -fn open_walk(canonical: &Path, avoid: &[(u64, u64)]) -> Result { - use std::os::unix::ffi::OsStrExt; - use std::os::unix::fs::MetadataExt; - let step = |dir: Option<&std::fs::File>, name: &[u8], flags: libc::c_int| { - let name = std::ffi::CString::new(name).map_err(|_| NO_SUCH_FILE.to_string())?; - let file = openat(dir, &name, flags).map_err(|e| match e.raw_os_error() { - Some(libc::ELOOP | libc::ENOTDIR | libc::ENOENT) => CHANGED.to_string(), - _ => cant_read(&e), - })?; - let meta = file.metadata().map_err(|e| cant_read(&e))?; - if avoid.contains(&(meta.dev(), meta.ino())) { - return Err(CHANGED.to_string()); - } - Ok(file) - }; - let mut names: Vec<&[u8]> = Vec::new(); - for c in canonical.components() { - match c { - Component::RootDir => {} - Component::Normal(n) => names.push(n.as_bytes()), - // A canonical path has no `.`, `..` or prefix. - _ => return Err(CHANGED.to_string()), - } - } - let (file, folders) = names.split_last().ok_or_else(|| NOT_REGULAR.to_string())?; - let mut dir = step(None, b"/", libc::O_RDONLY | libc::O_DIRECTORY)?; - for name in folders { - dir = step( - Some(&dir), - name, - libc::O_RDONLY | libc::O_DIRECTORY | libc::O_NOFOLLOW, - )?; - } - step( - Some(&dir), - file, - libc::O_RDONLY | libc::O_NOFOLLOW | libc::O_NONBLOCK, - ) -} - -/// `openat(2)` relative to `dir` (or the working folder, for an absolute -/// `name`), close-on-exec. -#[cfg(unix)] -fn openat( - dir: Option<&std::fs::File>, - name: &std::ffi::CStr, - flags: libc::c_int, -) -> std::io::Result { - use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; - let at = dir.map_or(libc::AT_FDCWD, |d| d.as_raw_fd()); - // SAFETY: `name` is a valid NUL-terminated string that outlives the - // call, and `at` is AT_FDCWD or a descriptor `dir` keeps open for it. - let fd = unsafe { libc::openat(at, name.as_ptr(), flags | libc::O_CLOEXEC) }; - // openat(2) returns -1 on error, else the new descriptor. - if fd == -1 { - return Err(std::io::Error::last_os_error()); - } - // SAFETY: openat just returned this descriptor, open and owned by no - // one else. - Ok(std::fs::File::from(unsafe { OwnedFd::from_raw_fd(fd) })) -} - -#[cfg(not(unix))] -fn open_walk(canonical: &Path, _avoid: &[(u64, u64)]) -> Result { - std::fs::File::open(canonical).map_err(|e| cant_read(&e)) -} - -/// INVARIANT 6: the opened handle is the file that was checked. -#[cfg(unix)] -fn same_file(checked: &std::fs::Metadata, opened: &std::fs::Metadata) -> Result<(), String> { - use std::os::unix::fs::MetadataExt; - // The inode number alone isn't the file: once a file is deleted, its - // number may go to the next one made (found on CI's filesystem, - // 2026-10-17). Its size and times tell the two apart; a rename, a new - // link or a write changes the change time. - let id = |m: &std::fs::Metadata| { - ( - m.dev(), - m.ino(), - m.size(), - (m.mtime(), m.mtime_nsec()), - (m.ctime(), m.ctime_nsec()), - ) - }; - if id(checked) == id(opened) { - Ok(()) - } else { - Err(CHANGED.to_string()) - } -} - -#[cfg(not(unix))] -fn same_file(_checked: &std::fs::Metadata, _opened: &std::fs::Metadata) -> Result<(), String> { - Ok(()) -} - -/// A file that changed between its check and its open. -pub const CHANGED: &str = "that file changed while it was being opened (a folder on its path moved or became a link): try again"; - -/// Every path that is missing, a dangling link, or outside the import -/// folders (INVARIANT 2). -pub const NO_SUCH_FILE: &str = "no such file in the server's import folders (by default your downloads folder): save it there first"; - -/// Every read when the server has no import folder (INVARIANT 0). -pub const NO_ROOTS: &str = "the books server has no import folder to read from (by default your downloads folder, which it can't use here): file it as base64 instead, or ask the human to set up an import folder for the books server"; - -/// A path into the books' or the vault's own files (INVARIANT 3). -pub const SEALED: &str = "that file belongs to the books or the TIN vault, which are never imported: these tools are the only way to them"; - -/// Any other file beside the books (INVARIANT 3). -pub const IN_THE_BOOKS: &str = "that file is one of the books' own, which are never read in place: save a copy of it in the server's import folders first"; - -/// A file with more than one hard link (INVARIANT 4). -pub const HARD_LINKED: &str = "that file has more than one hard link: save a plain copy of it in the server's import folders first"; - -/// A folder, pipe, or device. -pub const NOT_REGULAR: &str = "not a regular file (a folder, pipe, or device can't be imported)"; - -/// A read failure, without the path (it may be where a symlink led). -fn cant_read(e: &std::io::Error) -> String { - format!("can't read that file: {}", e.kind()) -} - -#[cfg(unix)] -fn hard_linked(meta: &std::fs::Metadata) -> bool { - use std::os::unix::fs::MetadataExt; - meta.nlink() > 1 -} - -#[cfg(not(unix))] -fn hard_linked(_meta: &std::fs::Metadata) -> bool { - false -} - -/// `path` made absolute and resolved as the kernel would follow it, name -/// by name: every symlink followed (a dangling one too, to where it -/// points), `.` and `..` folded in order, and whatever doesn't exist kept -/// as written. For an existing path this is its canonical form; for a -/// missing one, where it would be. -pub fn resolved(path: &Path) -> PathBuf { - let abs = if path.is_absolute() { - path.to_path_buf() - } else { - std::env::current_dir() - .unwrap_or_else(|_| PathBuf::from("/")) - .join(path) - }; - // The names still to walk, last first. - let mut todo: Vec = Vec::new(); - let mut out = PathBuf::new(); - // An absolute path starts the walk over at its root. - let push_all = |todo: &mut Vec, p: &Path, out: &mut PathBuf| { - *out = PathBuf::new(); - let mut parts = Vec::new(); - for c in p.components() { - match c { - Component::Prefix(_) | Component::RootDir => out.push(c.as_os_str()), - other => parts.push(other.as_os_str().to_os_string()), - } - } - todo.extend(parts.into_iter().rev()); - }; - push_all(&mut todo, &abs, &mut out); - // Links followed at most this many times (as the kernel's ELOOP). - let mut hops = 0; - while let Some(name) = todo.pop() { - if name == "." { - continue; - } - if name == ".." { - out.pop(); - continue; - } - let next = out.join(&name); - let link = std::fs::symlink_metadata(&next) - .is_ok_and(|m| m.file_type().is_symlink()) - .then(|| std::fs::read_link(&next).ok()) - .flatten(); - match link { - Some(target) if hops < 40 => { - hops += 1; - if target.is_absolute() { - push_all(&mut todo, &target, &mut out); - } else { - todo.extend( - target - .components() - .map(|c| c.as_os_str().to_os_string()) - .rev(), - ); - } - } - _ => out = next, - } - } - out -} - -#[cfg(all(test, unix))] -mod tests { - use super::*; - use proptest::prelude::*; - use std::collections::HashMap; - use std::os::unix::fs::symlink; - - /// A tree like a real machine's: a downloads root, a temp root holding - /// the books (data dir, vault) and the vault key's dir, a folder - /// outside every root, and a sealed folder outside every root (the - /// default key folder). `allowed` maps each canonical file path to - /// whether it may be imported: the oracle, written down by hand. - struct Tree { - base: PathBuf, - roots: ImportRoots, - protected: Protected, - allowed: HashMap, - } - - impl Drop for Tree { - fn drop(&mut self) { - let _ = std::fs::remove_dir_all(&self.base); - } - } - - /// A test's dir, removed when the test ends, pass or panic. - struct TestDir(PathBuf); - - impl Drop for TestDir { - fn drop(&mut self) { - let _ = std::fs::remove_dir_all(&self.0); - } - } - - fn tree(tag: &str) -> Tree { - let base = std::env::temp_dir().join(format!( - "ledger-mcp-local-files-{}-{tag}", - std::process::id() - )); - let _ = std::fs::remove_dir_all(&base); - let dl = base.join("Downloads"); - let tmp = base.join("tmp"); - let books = tmp.join("books"); - let data = books.join("books-data"); - let vault = books.join("vault"); - let keys = tmp.join("keys"); - let out = base.join("elsewhere"); - let secret = base.join("secret"); - for d in [ - secret.clone(), - dl.join("sub"), - dl.join("dir"), - data.join("blobs"), - vault.clone(), - keys.clone(), - out.clone(), - ] { - std::fs::create_dir_all(&d).unwrap(); - } - let write = |p: &Path| std::fs::write(p, b"bytes").unwrap(); - let mut allowed = HashMap::new(); - for (p, ok) in [ - (dl.join("receipt.pdf"), true), - (dl.join("sub/scan.png"), true), - (tmp.join("shot.png"), true), - (books.join("wanderbooks.toml"), false), - (data.join("journal.jsonl"), false), - (data.join("blobs/ab12"), false), - (vault.join("records.json"), false), - (keys.join("vault.key"), false), - (out.join("other.pdf"), false), - (secret.join("vault.key"), false), - ] { - write(&p); - allowed.insert(p.canonicalize().unwrap(), ok); - } - symlink(keys.join("vault.key"), dl.join("key-link")).unwrap(); - symlink(dl.join("receipt.pdf"), dl.join("good-link")).unwrap(); - symlink(out.join("other.pdf"), dl.join("out-link")).unwrap(); - symlink(&data, dl.join("data-link")).unwrap(); - std::fs::hard_link(keys.join("vault.key"), tmp.join("hard.key")).unwrap(); - allowed.insert(tmp.join("hard.key").canonicalize().unwrap(), false); - let fifo = dl.join("fifo"); - let made = std::process::Command::new("mkfifo") - .arg(&fifo) - .status() - .map(|s| s.success()) - .unwrap_or(false); - if made { - allowed.insert(fifo.canonicalize().unwrap(), false); - } - Tree { - roots: ImportRoots::new(vec![dl, tmp]), - protected: Protected { - sealed: vec![data, vault, keys, secret], - books: vec![books], - }, - base, - allowed, - } - } - - impl Tree { - fn check(&self, p: &Path) -> Result { - self.roots.check(p.to_str().unwrap(), &self.protected) - } - } - - #[test] - fn import_local_file_refuses_key_and_journal() { - let t = tree("refuses"); - let b = &t.base; - let refused = [ - (b.join("tmp/keys/vault.key"), "belongs to the books"), - ( - b.join("tmp/books/books-data/journal.jsonl"), - "belongs to the books", - ), - ( - b.join("tmp/books/books-data/blobs/ab12"), - "belongs to the books", - ), - (b.join("tmp/books/wanderbooks.toml"), "save a copy"), - (b.join("Downloads/key-link"), "belongs to the books"), - ( - b.join("Downloads/data-link/journal.jsonl"), - "belongs to the books", - ), - ( - b.join("Downloads/../tmp/books/books-data/journal.jsonl"), - "no such file", - ), - (b.join("tmp/hard.key"), "more than one hard link"), - (b.join("Downloads/dir"), "not a regular file"), - (b.join("Downloads"), "no such file"), - (b.join("Downloads/out-link"), "no such file"), - (b.join("Downloads/../elsewhere/other.pdf"), "no such file"), - (b.join("Downloads/missing.pdf"), "no such file"), - ]; - for (p, why) in &refused { - let err = t - .check(p) - .expect_err(&format!("{} must be refused", p.display())); - assert!(err.contains(why), "{}: {err}", p.display()); - // INVARIANT 5: the refusal never names a protected path. - assert!(!err.contains(b.to_str().unwrap()), "{err}"); - } - if t.allowed.keys().any(|p| p.ends_with("fifo")) { - let err = t.check(&b.join("Downloads/fifo")).unwrap_err(); - assert!(err.contains("not a regular file"), "{err}"); - } - assert!( - t.check(Path::new("relative.pdf")) - .unwrap_err() - .contains("absolute") - ); - for ok in [ - b.join("Downloads/receipt.pdf"), - b.join("Downloads/good-link"), - b.join("Downloads/sub/scan.png"), - b.join("tmp/shot.png"), - ] { - assert!(t.check(&ok).is_ok(), "{} must be allowed", ok.display()); - } - let (at, bytes) = t - .roots - .read( - b.join("Downloads/good-link").to_str().unwrap(), - &t.protected, - 5, - ) - .unwrap(); - assert_eq!(bytes, b"bytes"); - assert!(at.ends_with("Downloads/receipt.pdf")); - assert!( - t.roots - .read( - b.join("Downloads/receipt.pdf").to_str().unwrap(), - &t.protected, - 4 - ) - .unwrap_err() - .contains("limit") - ); - let _ = std::fs::remove_dir_all(b); - } - - /// Any absolute path built from the tree's names, `.` and `..`, and - /// its symlinks is allowed exactly when it has no `.` or `..` name and - /// resolves to a file the oracle allows: a regular file inside a root - /// and outside every protected dir. (One tree for every case, removed - /// when the test ends.) - #[test] - fn import_allowed_iff_regular_file_inside_a_root_outside_the_books() { - let t = tree("prop"); - let starts = [ - "Downloads", - "tmp", - "tmp/books", - "tmp/books/books-data", - "tmp/keys", - "elsewhere", - "secret", - ]; - let names = [ - "..", - ".", - "sub", - "dir", - "receipt.pdf", - "scan.png", - "shot.png", - "journal.jsonl", - "vault.key", - "key-link", - "good-link", - "data-link", - "hard.key", - "fifo", - ]; - proptest!(ProptestConfig::with_cases(512), |( - start in 0usize..7, - parts in proptest::collection::vec(0usize..14, 0..5), - )| { - let mut p = t.base.join(starts[start]); - for i in parts { - p.push(names[i]); - } - // A `.` or `..` name is refused before anything is looked up. - let dotted = p.to_str().unwrap().split('/').any(|n| n == "." || n == ".."); - let want = match p.canonicalize() { - Ok(c) if !dotted => t.allowed.get(&c).copied().unwrap_or(false), - _ => false, - }; - prop_assert_eq!(t.check(&p).is_ok(), want, "{}", p.display()); - }); - } - - /// INVARIANT 2: a path that is missing, a dangling link, or outside - /// the roots gets the same one refusal, whether a file is there or - /// not, sealed or not, and whether a link's target exists or not, so - /// a refusal can't be used to look around. Inside a root, sealed - /// places are refused as the books' own (existing or not, directly or - /// through a link, dangling or not), and the rest of the books dir is - /// filed from a copy. - #[test] - fn refusals_never_tell_what_is_there() { - let t = tree("probe"); - let b = &t.base; - let dl = b.join("Downloads"); - symlink(b.join("elsewhere/missing.pdf"), dl.join("dangling-out")).unwrap(); - symlink(dl.join("missing.pdf"), dl.join("dangling-in")).unwrap(); - symlink(b.join("tmp/keys/missing.key"), dl.join("dangling-key")).unwrap(); - symlink(b.join("tmp/keys/vault.key"), dl.join("key-link-2")).unwrap(); - for p in [ - dl.join("missing.pdf"), - dl.join("dangling-in"), - dl.join("dangling-out"), - b.join("elsewhere/other.pdf"), - b.join("elsewhere/missing.pdf"), - b.join("elsewhere"), - b.join("secret/vault.key"), - b.join("secret/missing.key"), - b.join("no-such-dir/x.pdf"), - b.join("Downloads/out-link"), - b.join("Downloads/../secret/vault.key"), - b.join("Downloads/missing/../../secret/vault.key"), - ] { - assert_eq!( - t.check(&p), - Err(NO_SUCH_FILE.to_string()), - "{}", - p.display() - ); - } - for p in [ - b.join("tmp/books/books-data/journal.jsonl"), - b.join("tmp/books/books-data/no-such-line"), - b.join("tmp/books/vault/records.json"), - b.join("tmp/keys/vault.key"), - b.join("tmp/keys/missing.key"), - dl.join("key-link-2"), - dl.join("dangling-key"), - ] { - assert_eq!(t.check(&p), Err(SEALED.to_string()), "{}", p.display()); - } - for p in [ - b.join("tmp/books/wanderbooks.toml"), - b.join("tmp/books/company/missing.pdf"), - ] { - assert_eq!( - t.check(&p), - Err(IN_THE_BOOKS.to_string()), - "{}", - p.display() - ); - } - let _ = std::fs::remove_dir_all(b); - } - - /// INVARIANT 2: a path with a `.` or `..` name is refused as missing - /// before anything is looked up, so it can't tell whether a folder - /// outside the roots exists (`a/b/..` resolves only when `b` is a - /// folder), even when it would lead back to a file the roots allow. - #[test] - fn a_dot_or_dotdot_name_never_tells_whether_folders_outside_exist() { - let t = tree("dots"); - let b = &t.base; - let base = b.to_str().unwrap(); - for p in [ - format!("{base}/Downloads/../elsewhere/../Downloads/receipt.pdf"), - format!("{base}/Downloads/../no-such-dir/../Downloads/receipt.pdf"), - format!("{base}/elsewhere/../Downloads/receipt.pdf"), - format!("{base}/no-such-dir/../Downloads/receipt.pdf"), - format!("{base}/Downloads/sub/../receipt.pdf"), - format!("{base}/Downloads/./receipt.pdf"), - format!("{base}/Downloads/receipt.pdf/."), - format!("{base}/Downloads/sub/.."), - format!("{base}/tmp/books/books-data/../books-data/journal.jsonl"), - format!("{base}/tmp/keys/./vault.key"), - ] { - assert_eq!( - t.roots.check(&p, &t.protected), - Err(NO_SUCH_FILE.to_string()), - "{p}" - ); - assert_eq!( - t.roots.read(&p, &t.protected, 1 << 20).unwrap_err(), - NO_SUCH_FILE, - "{p}" - ); - } - // The same file, named plainly, is read. - assert!( - t.roots - .check(&format!("{base}/Downloads/receipt.pdf"), &t.protected) - .is_ok() - ); - // Names that merely contain dots are names. - std::fs::write(b.join("Downloads/..receipt.pdf"), b"bytes").unwrap(); - std::fs::write(b.join("Downloads/receipt..pdf"), b"bytes").unwrap(); - for name in ["..receipt.pdf", "receipt..pdf"] { - assert!( - t.roots - .check(&format!("{base}/Downloads/{name}"), &t.protected) - .is_ok(), - "{name}" - ); - } - let _ = std::fs::remove_dir_all(b); - } - - /// INVARIANT 6: a file swapped after the check is never read. Another - /// file renamed over it differs by inode; one that took the deleted - /// file's inode number differs by its size and times; a symlink is - /// never followed; a pipe neither blocks the open nor is read. - #[test] - fn a_file_swapped_after_the_check_is_never_read() { - let t = tree("swap"); - let dl = t.base.join("Downloads"); - let target = dl.join("swapped.pdf"); - let check = |what: &str| { - std::fs::write(&target, b"checked bytes").unwrap(); - inspect(&t.roots.usable(), target.to_str().unwrap(), &t.protected) - .unwrap_or_else(|e| panic!("{what}: {e}")) - }; - - // Another file, made while the checked one still exists (so its - // own inode), renamed over it. - let (canonical, meta) = check("rename"); - std::fs::write(dl.join("other.pdf"), b"other bytes").unwrap(); - std::fs::rename(dl.join("other.pdf"), &target).unwrap(); - assert_eq!( - read_checked(&canonical, &meta, &t.protected.inodes(), 100), - Err(CHANGED.to_string()) - ); - std::fs::remove_file(&target).unwrap(); - - // The checked file deleted and another made in its place, which - // a filesystem may give the same inode number: its device and - // inode match, and the file is still another. - let (canonical, meta) = check("reused inode"); - std::fs::remove_file(&target).unwrap(); - std::fs::write(&target, b"other bytes").unwrap(); - assert_eq!( - read_checked(&canonical, &meta, &t.protected.inodes(), 100), - Err(CHANGED.to_string()) - ); - std::fs::remove_file(&target).unwrap(); - // The same inode, written after the check: another file now. - let (canonical, meta) = check("rewritten"); - std::fs::write(&target, b"rewritten bytes").unwrap(); - assert_eq!( - read_checked(&canonical, &meta, &t.protected.inodes(), 100), - Err(CHANGED.to_string()) - ); - std::fs::remove_file(&target).unwrap(); - let (canonical, meta) = check("unchanged"); - assert_eq!( - read_checked(&canonical, &meta, &t.protected.inodes(), 100), - Ok(b"checked bytes".to_vec()), - "the checked file itself is read" - ); - std::fs::remove_file(&target).unwrap(); - - let (canonical, meta) = check("symlink"); - std::fs::remove_file(&target).unwrap(); - symlink(t.base.join("tmp/keys/vault.key"), &target).unwrap(); - assert_eq!( - read_checked(&canonical, &meta, &t.protected.inodes(), 100), - Err(CHANGED.to_string()) - ); - std::fs::remove_file(&target).unwrap(); - - let (canonical, meta) = check("fifo"); - std::fs::remove_file(&target).unwrap(); - let made = std::process::Command::new("mkfifo") - .arg(&target) - .status() - .is_ok_and(|s| s.success()); - if made { - let (tx, rx) = std::sync::mpsc::channel(); - let inodes = t.protected.inodes(); - std::thread::spawn(move || tx.send(read_checked(&canonical, &meta, &inodes, 100))); - let got = rx - .recv_timeout(std::time::Duration::from_secs(10)) - .expect("opening a pipe blocked"); - assert_eq!(got, Err(NOT_REGULAR.to_string())); - } - let _ = std::fs::remove_dir_all(&t.base); - } - - /// Where a file could be read back in: strictly inside a root, - /// existing or not; a root itself, or a path outside, is not. - #[test] - fn contains_is_strictly_inside_a_root() { - let t = tree("contains"); - let b = &t.base; - assert!(t.roots.contains(&b.join("Downloads/receipt.pdf"))); - assert!(t.roots.contains(&b.join("Downloads/new/file.zip"))); - assert!(t.roots.contains(&b.join("Downloads/good-link"))); - assert!(!t.roots.contains(&b.join("Downloads"))); - assert!(!t.roots.contains(&b.join("elsewhere/x.zip"))); - assert!(!t.roots.contains(&b.join("Downloads/../elsewhere/x.zip"))); - assert!(!t.roots.contains(&b.join("Downloads/out-link"))); - let _ = std::fs::remove_dir_all(b); - } - - /// INVARIANT 6, the reviewer's race (P0 review round 2): a folder on - /// the checked path is swapped for a link to the data dir after the - /// check, and the file's metadata is taken after the swap (so its - /// device and inode are the journal's). The walk refuses the link, - /// with no help from the kernel's /proc paths; and a protected folder - /// renamed into place (no link to refuse) is refused by its inode. - #[test] - fn a_folder_swapped_after_the_check_is_never_walked_through() { - let t = tree("swap-dir"); - let dl = t.base.join("Downloads"); - let data = t.base.join("tmp/books/books-data"); - std::fs::write(dl.join("sub/journal.jsonl"), b"decoy").unwrap(); - let given = dl.join("sub/journal.jsonl"); - let canonical = t.check(&given).unwrap(); - // Taken as a read takes them: before the check. - let avoid = t.protected.inodes(); - - // A link: the walk fails at it (ELOOP or ENOTDIR). - std::fs::rename(dl.join("sub"), dl.join("sub.old")).unwrap(); - symlink(&data, dl.join("sub")).unwrap(); - let meta = std::fs::metadata(&canonical).unwrap(); - assert_eq!( - read_checked(&canonical, &meta, &avoid, 1000), - Err(CHANGED.to_string()) - ); - std::fs::remove_file(dl.join("sub")).unwrap(); - - // The data dir itself renamed into place: every name on the walk - // is a real folder, but one of them is protected. - std::fs::rename(&data, dl.join("sub")).unwrap(); - let meta = std::fs::metadata(&canonical).unwrap(); - assert_eq!( - read_checked(&canonical, &meta, &avoid, 1000), - Err(CHANGED.to_string()) - ); - std::fs::rename(dl.join("sub"), &data).unwrap(); - std::fs::rename(dl.join("sub.old"), dl.join("sub")).unwrap(); - - // Untouched, the same walk reads the checked file. - let meta = std::fs::metadata(&canonical).unwrap(); - assert_eq!( - read_checked(&canonical, &meta, &avoid, 1000).unwrap(), - b"decoy" - ); - let _ = std::fs::remove_dir_all(&t.base); - } - - /// INVARIANT 6, links and pipes on the walk: a folder on the checked - /// path swapped for a link to a folder outside every root (nothing - /// protected there, and the file's metadata taken after the swap, so - /// it is the other file's) is never walked through; a folder swapped - /// for a pipe neither blocks the walk nor is opened. - #[test] - fn a_folder_swapped_for_a_link_or_a_pipe_is_never_walked_through() { - let t = tree("swap-link"); - let dl = t.base.join("Downloads"); - let out = t.base.join("elsewhere"); - std::fs::write(dl.join("sub/other.pdf"), b"checked").unwrap(); - let canonical = t.check(&dl.join("sub/other.pdf")).unwrap(); - let checked = std::fs::metadata(&canonical).unwrap(); - let avoid = t.protected.inodes(); - std::fs::rename(dl.join("sub"), dl.join("sub.old")).unwrap(); - - symlink(&out, dl.join("sub")).unwrap(); - let meta = std::fs::metadata(&canonical).unwrap(); - assert_eq!( - std::fs::read(&canonical).unwrap(), - b"bytes", - "the link leads to the other file" - ); - assert_eq!( - read_checked(&canonical, &meta, &avoid, 1000), - Err(CHANGED.to_string()) - ); - std::fs::remove_file(dl.join("sub")).unwrap(); - - let made = std::process::Command::new("mkfifo") - .arg(dl.join("sub")) - .status() - .is_ok_and(|s| s.success()); - if made { - let (tx, rx) = std::sync::mpsc::channel(); - let (c, a) = (canonical.clone(), avoid.clone()); - std::thread::spawn(move || tx.send(read_checked(&c, &checked, &a, 1000))); - let got = rx - .recv_timeout(std::time::Duration::from_secs(10)) - .expect("walking onto a pipe blocked"); - assert_eq!(got, Err(CHANGED.to_string())); - } - let _ = std::fs::remove_dir_all(&t.base); - } - - /// A read failure names what went wrong, never the path. - #[test] - fn cant_read_names_the_failure_not_the_path() { - let e = std::io::Error::new(std::io::ErrorKind::PermissionDenied, "/home/x/secret"); - assert_eq!(cant_read(&e), "can't read that file: permission denied"); - } - - /// `resolved` follows as many links as the kernel does (40, as - /// `canonicalize` does), and no more. - #[test] - fn resolved_follows_forty_links_and_no_more() { - let t = tree("hops"); - let b = t.base.canonicalize().unwrap(); - let file = b.join("Downloads/receipt.pdf"); - // link-n leads to link-(n-1), ..., link-1 to the file: n links. - let link = |n: usize| b.join(format!("Downloads/link-{n}")); - symlink(&file, link(1)).unwrap(); - for n in 2..=41 { - symlink(link(n - 1), link(n)).unwrap(); - } - assert_eq!(link(40).canonicalize().unwrap(), file); - assert_eq!(resolved(&link(40)), file); - assert!(link(41).canonicalize().is_err(), "the kernel stops at 40"); - assert_ne!(resolved(&link(41)), file); - let _ = std::fs::remove_dir_all(&t.base); - } - - /// `resolved` places a path where the kernel would follow it: the - /// canonical form for one that exists, and through a dangling link to - /// where it points. - #[test] - fn resolved_follows_every_link_a_dangling_one_too() { - let t = tree("resolved"); - let b = t.base.canonicalize().unwrap(); - for p in [ - b.join("Downloads/good-link"), - b.join("Downloads/data-link/journal.jsonl"), - b.join("Downloads/sub/../receipt.pdf"), - b.join("Downloads/./key-link"), - ] { - assert_eq!(resolved(&p), p.canonicalize().unwrap(), "{}", p.display()); - } - symlink("../elsewhere/gone.pdf", b.join("Downloads/rel-dangling")).unwrap(); - assert_eq!( - resolved(&b.join("Downloads/rel-dangling")), - b.join("elsewhere/gone.pdf") - ); - symlink(b.join("Downloads/loop-b"), b.join("Downloads/loop-a")).unwrap(); - symlink(b.join("Downloads/loop-a"), b.join("Downloads/loop-b")).unwrap(); - // A loop ends (the hop limit) inside the folder it loops in. - assert!(resolved(&b.join("Downloads/loop-a")).starts_with(b.join("Downloads"))); - assert_eq!( - resolved(&b.join("Downloads/missing/../../elsewhere/x")), - b.join("elsewhere/x") - ); - let _ = std::fs::remove_dir_all(&t.base); - } - - /// INVARIANT 0, the default root: the downloads folder, never `/` or - /// the home folder (an XDG_DOWNLOAD_DIR of "$HOME/", "$HOME", "/" - /// falls back to ~/Downloads), and ~/Downloads only when it is a real - /// folder that isn't the home folder; else no roots, and every read - /// says so. - #[test] - fn the_default_root_is_never_the_home_folder_or_the_filesystem() { - let base = std::env::temp_dir().join(format!( - "ledger-mcp-local-files-{}-defaults", - std::process::id() - )); - let _ = std::fs::remove_dir_all(&base); - let _cleanup = TestDir(base.clone()); - let home = base.join("home"); - std::fs::create_dir_all(home.join(".ssh")).unwrap(); - std::fs::write(home.join(".ssh/id_ed25519"), b"PRIVATE KEY").unwrap(); - let home = home.canonicalize().unwrap(); - let os = |s: &str| Some(std::ffi::OsString::from(s)); - let roots = |xdg: Option| { - ImportRoots::defaults_from(xdg, Some(home.clone())).usable() - }; - // No ~/Downloads: nothing to fall back to. - for xdg in [ - os("$HOME/"), - os("$HOME"), - os("${HOME}"), - os("/"), - os("Downloads"), - None, - ] { - assert_eq!(roots(xdg.clone()), Vec::::new(), "{xdg:?}"); - } - let err = ImportRoots::defaults_from(os("$HOME/"), Some(home.clone())) - .read( - home.join(".ssh/id_ed25519").to_str().unwrap(), - &Protected::default(), - 100, - ) - .unwrap_err(); - assert_eq!(err, NO_ROOTS); - // ~/Downloads a link to the home folder: still the home folder. - symlink(&home, home.join("Downloads")).unwrap(); - assert!(roots(None).is_empty()); - std::fs::remove_file(home.join("Downloads")).unwrap(); - // A real ~/Downloads: the fallback, and the default. - std::fs::create_dir_all(home.join("Downloads")).unwrap(); - let downloads = vec![home.join("Downloads")]; - for xdg in [ - os("$HOME/"), - os("$HOME"), - os("/"), - os("$HOME/Downloads"), - None, - ] { - assert_eq!(roots(xdg.clone()), downloads, "{xdg:?}"); - } - // Another real folder named by XDG_DOWNLOAD_DIR wins. - std::fs::create_dir_all(home.join("Inbox")).unwrap(); - assert_eq!(roots(os("$HOME/Inbox")), vec![home.join("Inbox")]); - // Without HOME, "$HOME/..." means nothing. - assert!( - ImportRoots::defaults_from(os("$HOME/Inbox"), None) - .usable() - .is_empty() - ); - let _ = std::fs::remove_dir_all(&base); - } - - /// INVARIANT 0, a named root: `/`, the home folder, a folder holding - /// it, and a folder holding the books, the vault or a key are refused - /// (with why, never silently); anything else is kept. - #[test] - fn a_root_holding_the_books_or_the_home_folder_is_refused() { - let t = tree("guard"); - let b = t.base.canonicalize().unwrap(); - let home = b.join("elsewhere"); - let keep_out = vec![ - b.join("tmp/books/books-data"), - b.join("tmp/books"), - b.join("tmp/keys/vault.key"), - ]; - let roots = ImportRoots::new(vec![ - PathBuf::from("/"), - home.clone(), - b.clone(), - b.join("tmp"), - b.join("tmp/keys"), - b.join("Downloads"), - b.join("no-such-root"), - b.join("tmp/books/books-data/blobs"), - b.join("tmp/books/vault"), - ]); - let (kept, refused) = roots.guard(Some(&home), &keep_out); - assert!(kept.explicit()); - assert_eq!(kept.usable(), vec![b.join("Downloads")]); - let refused: Vec<(PathBuf, &str)> = refused; - assert_eq!( - refused.iter().map(|(p, _)| p.clone()).collect::>(), - vec![ - PathBuf::from("/"), - home.clone(), - b.clone(), - b.join("tmp"), - b.join("tmp/keys"), - b.join("tmp/books/books-data/blobs"), - b.join("tmp/books/vault"), - ] - ); - assert!(refused[0].1.contains("whole filesystem")); - assert!(refused[1].1.contains("home folder")); - assert!(refused[3].1.contains("holds the books")); - assert!( - refused[5].1.contains("inside the books"), - "{}", - refused[5].1 - ); - assert!( - refused[6].1.contains("inside the books"), - "{}", - refused[6].1 - ); - let _ = std::fs::remove_dir_all(&t.base); - } - - /// Protected::of INVARIANTS 1-2: the key files are sealed wherever - /// they are, with where they lead; their folders only when sealing - /// one wouldn't seal a whole root (the home folder, a root, a folder - /// holding one), so a key kept beside receipts doesn't block every - /// import. ~/.config/wanderbooks is sealed whatever XDG_CONFIG_HOME - /// says. - #[test] - fn sealing_a_key_never_seals_a_whole_root() { - let t = tree("seal-key"); - let b = t.base.canonicalize().unwrap(); - let dl = b.join("Downloads"); - let home = b.join("home"); - std::fs::create_dir_all(home.join(".config/wanderbooks")).unwrap(); - std::fs::write(home.join(".config/wanderbooks/vault.key"), b"old key").unwrap(); - std::fs::write(home.join("vault.key"), b"key in home").unwrap(); - std::fs::write(dl.join("vault.key"), b"key in a root").unwrap(); - std::fs::write(dl.join("sub/vault.key"), b"key in a folder").unwrap(); - let roots = vec![dl.clone(), home.join("Downloads")]; - let protected = |key: PathBuf| { - Protected::of( - &Places { - data_dir: b.join("tmp/books/books-data"), - books_dir: b.join("tmp/books"), - vault_dir: b.join("tmp/books/vault"), - key_files: vec![key, b.join("xdg/wanderbooks/vault.key")], - home: Some(home.clone()), - }, - &roots, - ) - }; - let r = ImportRoots::new(vec![dl.clone()]); - let check = |key: PathBuf, path: PathBuf| r.check(path.to_str().unwrap(), &protected(key)); - // In a root: the key is sealed, the root isn't. - assert_eq!( - check(dl.join("vault.key"), dl.join("vault.key")), - Err(SEALED.into()) - ); - assert!(check(dl.join("vault.key"), dl.join("receipt.pdf")).is_ok()); - // In a folder inside a root: that folder is sealed, the root isn't. - assert_eq!( - check(dl.join("sub/vault.key"), dl.join("sub/scan.png")), - Err(SEALED.into()) - ); - assert!(check(dl.join("sub/vault.key"), dl.join("receipt.pdf")).is_ok()); - // In the home folder: the home folder (holding a root) isn't. - let p = protected(home.join("vault.key")); - assert!(!p.sealed.contains(&home)); - assert!(p.sealed.contains(&home.join("vault.key"))); - // A link to a key: the link and its target are both sealed (the - // target is outside the root here, so a path to it is refused - // first as outside: INVARIANT 2). - assert_eq!( - check(dl.join("key-link"), dl.join("key-link")), - Err(NO_SUCH_FILE.into()) - ); - let p = protected(dl.join("key-link")); - assert!(p.sealed.contains(&dl.join("key-link"))); - assert!(p.sealed.contains(&b.join("tmp/keys/vault.key"))); - assert!(p.sealed.contains(&b.join("tmp/keys"))); - // Always: ~/.config/wanderbooks, and the default key's folder. - assert!(p.sealed.contains(&home.join(".config/wanderbooks"))); - assert!(p.sealed.contains(&b.join("xdg/wanderbooks"))); - let _ = std::fs::remove_dir_all(&t.base); - } -} diff --git a/crates/ledger-mcp/src/paths.rs b/crates/ledger-mcp/src/paths.rs new file mode 100644 index 0000000..eb058a3 --- /dev/null +++ b/crates/ledger-mcp/src/paths.rs @@ -0,0 +1,132 @@ +//! Paths as the server judges them: where a path really leads. + +use std::path::{Component, Path, PathBuf}; + +/// `path` made absolute and resolved as the kernel would follow it, name +/// by name: every symlink followed (a dangling one too, to where it +/// points), `.` and `..` folded in order, and whatever doesn't exist kept +/// as written. For an existing path this is its canonical form; for a +/// missing one, where it would be. +pub fn resolved(path: &Path) -> PathBuf { + let abs = if path.is_absolute() { + path.to_path_buf() + } else { + std::env::current_dir() + .unwrap_or_else(|_| PathBuf::from("/")) + .join(path) + }; + // The names still to walk, last first. + let mut todo: Vec = Vec::new(); + let mut out = PathBuf::new(); + // An absolute path starts the walk over at its root. + let push_all = |todo: &mut Vec, p: &Path, out: &mut PathBuf| { + *out = PathBuf::new(); + let mut parts = Vec::new(); + for c in p.components() { + match c { + Component::Prefix(_) | Component::RootDir => out.push(c.as_os_str()), + other => parts.push(other.as_os_str().to_os_string()), + } + } + todo.extend(parts.into_iter().rev()); + }; + push_all(&mut todo, &abs, &mut out); + // Links followed at most this many times (as the kernel's ELOOP). + let mut hops = 0; + while let Some(name) = todo.pop() { + if name == "." { + continue; + } + if name == ".." { + out.pop(); + continue; + } + let next = out.join(&name); + let link = std::fs::symlink_metadata(&next) + .is_ok_and(|m| m.file_type().is_symlink()) + .then(|| std::fs::read_link(&next).ok()) + .flatten(); + match link { + Some(target) if hops < 40 => { + hops += 1; + if target.is_absolute() { + push_all(&mut todo, &target, &mut out); + } else { + todo.extend( + target + .components() + .map(|c| c.as_os_str().to_os_string()) + .rev(), + ); + } + } + _ => out = next, + } + } + out +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + use std::os::unix::fs::symlink; + + /// `resolved` follows as many links as the kernel does (40, as + /// `canonicalize` does), and no more. + #[test] + fn resolved_follows_forty_links_and_no_more() { + let dir = tempfile::tempdir().unwrap(); + let b = dir.path().canonicalize().unwrap(); + let file = b.join("receipt.pdf"); + std::fs::write(&file, "x").unwrap(); + let link = |n: usize| b.join(format!("link-{n}")); + symlink(&file, link(1)).unwrap(); + for n in 2..=41 { + symlink(link(n - 1), link(n)).unwrap(); + } + assert_eq!(link(40).canonicalize().unwrap(), file); + assert_eq!(resolved(&link(40)), file); + assert!(link(41).canonicalize().is_err(), "the kernel stops at 40"); + assert_ne!(resolved(&link(41)), file); + } + + /// `resolved` places a path where the kernel would follow it: the + /// canonical form for one that exists, and through a dangling link to + /// where it points. + #[test] + fn resolved_follows_every_link_a_dangling_one_too() { + let dir = tempfile::tempdir().unwrap(); + let b = dir.path().canonicalize().unwrap(); + std::fs::create_dir_all(b.join("Downloads/sub")).unwrap(); + std::fs::create_dir_all(b.join("data")).unwrap(); + std::fs::write(b.join("Downloads/receipt.pdf"), "x").unwrap(); + std::fs::write(b.join("data/journal.jsonl"), "x").unwrap(); + symlink( + b.join("Downloads/receipt.pdf"), + b.join("Downloads/good-link"), + ) + .unwrap(); + symlink(b.join("data"), b.join("Downloads/data-link")).unwrap(); + for p in [ + b.join("Downloads/good-link"), + b.join("Downloads/data-link/journal.jsonl"), + b.join("Downloads/sub/../receipt.pdf"), + b.join("Downloads/./good-link"), + ] { + assert_eq!(resolved(&p), p.canonicalize().unwrap(), "{}", p.display()); + } + symlink("../elsewhere/gone.pdf", b.join("Downloads/rel-dangling")).unwrap(); + assert_eq!( + resolved(&b.join("Downloads/rel-dangling")), + b.join("elsewhere/gone.pdf") + ); + symlink(b.join("Downloads/loop-b"), b.join("Downloads/loop-a")).unwrap(); + symlink(b.join("Downloads/loop-a"), b.join("Downloads/loop-b")).unwrap(); + // A loop ends (the hop limit) inside the folder it loops in. + assert!(resolved(&b.join("Downloads/loop-a")).starts_with(b.join("Downloads"))); + assert_eq!( + resolved(&b.join("Downloads/missing/../../elsewhere/x")), + b.join("elsewhere/x") + ); + } +} diff --git a/crates/ledger-mcp/src/scopes.rs b/crates/ledger-mcp/src/scopes.rs index 82c0980..b5a3718 100644 --- a/crates/ledger-mcp/src/scopes.rs +++ b/crates/ledger-mcp/src/scopes.rs @@ -119,7 +119,7 @@ pub const TABLE: [(&str, Needs); 109] = [ ("books.assign_role", P), ("books.set_tax_category", W), ("books.import_chart_csv", W), - ("books.import_local_file", W), + ("books.begin_upload", W), ("books.export", R), ("books.register_asset", W), ("books.asset_schedule", R), diff --git a/crates/ledger-mcp/src/server.rs b/crates/ledger-mcp/src/server.rs index 4bd96c6..dda5798 100644 --- a/crates/ledger-mcp/src/server.rs +++ b/crates/ledger-mcp/src/server.rs @@ -32,7 +32,7 @@ //! ``` use crate::handbook; -use crate::local_files::{ImportRoots, Places, Protected, resolved}; +use crate::paths::resolved; use crate::policy::ReceiptPolicy; use crate::settings::{Live, Settings}; use crate::tin_files; @@ -57,9 +57,6 @@ use rmcp::{ServerHandler, tool, tool_handler, tool_router}; use rust_decimal::Decimal; use std::sync::{Arc, Mutex}; -/// Max receipt/document upload: 25 MB. -const MAX_UPLOAD_BYTES: usize = 25 * 1024 * 1024; - /// The server. One store, one receipt policy, shared across tool calls. /// `Store` is `!Sync`, so access is serialized through a mutex; the server /// is single-writer by construction. Tool bodies (synchronous disk and @@ -69,12 +66,14 @@ const MAX_UPLOAD_BYTES: usize = 25 * 1024 * 1024; pub struct BooksServer { store: Arc, settings: Arc>, - import_roots: Arc, layout: Arc, /// The book's access log, for books served from the data root: every /// call is then checked against its caller's token and recorded /// (access.rs). None for a books dir served over stdio. access: Option>, + /// The book's uploads (books served from the data root; none over + /// stdio, where no bytes can be sent but inline). + uploads: Option, } /// Where a book's own files live around its data dir. @@ -157,6 +156,31 @@ pub struct SharedStore { } impl SharedStore { + /// An attachment's name, type and bytes (hash-verified), for a + /// download: blocking (the store's lock, then the file). + /// None when there is no such attachment. + pub fn attachment_file(&self, id: u64) -> Result)>, String> { + if self.stopped() { + return Err(STOPPED.to_string()); + } + let store = self.store.blocking_lock(); + let Some(meta) = store + .list_attachments(None) + .map_err(|e| e.to_string())? + .into_iter() + .find(|a| a.id == id) + else { + return Ok(None); + }; + let bytes = store.attachment_bytes(id).map_err(|e| e.to_string())?; + Ok(Some((meta.file_name, meta.mime_type, bytes))) + } + + /// Whether it stopped taking requests (INVARIANT 2). + pub fn is_stopped(&self) -> bool { + self.stopped() + } + /// Share an open store. pub fn new(store: Store) -> Arc { Arc::new(Self { @@ -200,12 +224,29 @@ impl SharedStore { } } -/// Why a file with full TINs isn't written into an import root. -const TIN_FILE_IN_ROOT: &str = "a file with full TINs is never saved in the server's import folders (by default your downloads folder), where it could be filed back into the books: save it elsewhere, such as your documents folder"; +/// A filed file's name and type: those given, else the upload's own. +fn named( + file_name: Option, + mime_type: Option, + file: &files::TakenFile, +) -> Result<(String, String), String> { + let name = file_name + .or_else(|| file.file_name.clone()) + .ok_or("file_name is needed")?; + let mime = mime_type + .or_else(|| file.mime.clone()) + .ok_or("mime_type is needed")?; + if let Some(why) = ledger_store::uploads::file_name_problem(&name) + .or_else(|| ledger_store::uploads::mime_problem(&mime)) + { + return Err(why); + } + Ok((name, mime)) +} /// What every request gets after an internal error (SharedStore /// INVARIANT 2). -const STOPPED: &str = "the books server stopped taking requests after an internal error, to protect the books: ask the human to restart the books server (the books are checked again when it starts)"; +pub(crate) const STOPPED: &str = "the books server stopped taking requests after an internal error, to protect the books: ask the human to restart the books server (the books are checked again when it starts)"; /// Why the vault won't open with a key file inside the books. const KEY_IN_BOOKS: &str = "the TIN vault's key file is inside the books' folder, beside the records it unlocks: ask the human to move it out of the books and set [vault] key_file in the settings"; @@ -555,12 +596,18 @@ impl BooksServer { policy, ..Settings::default() }))), - import_roots: Arc::new(ImportRoots::defaults()), layout: Arc::new(Layout::BooksDir), access: None, + uploads: None, } } + /// Take uploaded files from `uploads` (`books.begin_upload`). + pub fn with_uploads(mut self, uploads: ledger_store::uploads::Uploads) -> Self { + self.uploads = Some(uploads); + self + } + /// Check every call against its caller's token, and record it in /// `log` (access.rs). pub fn with_access(mut self, log: Arc) -> Self { @@ -575,12 +622,6 @@ impl BooksServer { self } - /// The folders `books.import_local_file` may read from (local_files.rs). - pub fn with_import_roots(mut self, roots: ImportRoots) -> Self { - self.import_roots = Arc::new(roots); - self - } - /// Take the settings from files, reloading them whenever they change /// (crate::settings), shared with the other sessions holding `live`. pub fn with_live_settings(mut self, live: Arc>) -> Self { @@ -655,7 +696,7 @@ impl BooksServer { /// Every tool the server serves. pub(crate) fn tool_router() -> rmcp::handler::server::router::tool::ToolRouter { - Self::core_router() + Self::kept_router() + Self::access_router() + Self::core_router() + Self::kept_router() + Self::access_router() + Self::files_router() } /// The settings in force for this request (reloaded at its start, in @@ -902,20 +943,8 @@ impl BooksServer { self.layout.vault_dir(store) } - /// What is never read through a path, even inside an import root - /// (local_files.rs INVARIANT 3; Protected::of): the data dir, the - /// vault, the key file in force and the default one (a key left there - /// from before still unlocks the vault), and the folders it seals; and - /// the rest of the books dir. - fn protected(&self, store: &Store) -> Protected { - Protected::of( - &places(store, &self.settings(), &self.layout), - &self.import_roots.usable(), - ) - } - /// Refuse bytes on their way into the books (books.attach_receipt, - /// books.file_document, books.import_local_file) if they carry a full + /// books.file_document, books.set_w9's file) if they carry a full /// TIN: a file the server read or wrote with TINs, or text holding one /// the vault holds or one printed as an SSN or EIN is (tin_files.rs /// INVARIANT 3), or text probing for one (INVARIANT 4). @@ -963,62 +992,78 @@ impl BooksServer { .collect() } - /// Read a file an agent names (books.import_local_file, books.set_w9 - /// w9_file_path) through the import roots' lockdown (local_files.rs), - /// with its canonical path. - fn read_local(&self, path: &str) -> Result<(std::path::PathBuf, Vec), String> { - let protected = { - let store = self.lock()?; - self.protected(&store) - }; - self.import_roots - .read(path.trim(), &protected, MAX_UPLOAD_BYTES) - } - /// Where a file holding full TINs may be written: a new file outside - /// the books (outside_books) and outside every import root, so it can - /// never be filed back in as a receipt and read out again. + /// the books (outside_books). (It can't be filed back in unnoticed: + /// tin_files.rs records it.) fn tin_file_dest(&self, store: &Store, path: &str) -> Result { - let dest = outside_books(&self.layout.books_dir(store), path)?; - self.outside_import_roots(&dest)?; - Ok(dest) + outside_books(&self.layout.books_dir(store), path) } - fn outside_import_roots(&self, dest: &std::path::Path) -> Result<(), String> { - if self.import_roots.contains(dest) { - return Err(TIN_FILE_IN_ROOT.to_string()); - } - Ok(()) - } - - /// Read a W-9 file (PDF, PNG, or JPEG, 25 MB at most) for the vault, - /// through the import roots' lockdown like any local file: the type - /// is judged by where the path really leads (a link named w9.pdf to - /// another file is that file). - fn read_w9_file(&self, path: &str) -> Result { - let (canonical, bytes) = self.read_local(path)?; + /// The signed W-9 a call gives (an upload, or inline), for the vault: + /// a PDF, PNG, or JPEG by its own first bytes, whatever it is called. + fn w9_file( + &self, + p: &SetW9Params, + ) -> Result<(ledger_vault::W9File, Option), String> { + let file = self.take_file(files::Given { + upload_id: p.w9_upload_id.as_deref(), + content_base64: p.w9_content_base64.as_deref(), + sha256: p.w9_sha256.as_deref(), + purposes: files::W9, + })?; // Not recorded here: set_w9 records it once the call can only // succeed, just before the vault write (tin_files.rs INVARIANT 1), // so a refused call never marks an ordinary file. - let mime_type = match canonical - .extension() - .and_then(|e| e.to_str()) - .map(str::to_lowercase) - .as_deref() - { - Some("pdf") => "application/pdf", - Some("png") => "image/png", - Some("jpg" | "jpeg") => "image/jpeg", - _ => return Err("w9_file_path is a PDF, PNG, or JPEG (.pdf, .png, .jpg)".into()), + let (mime_type, ext) = if file.bytes.starts_with(b"%PDF-") { + ("application/pdf", "pdf") + } else if file.bytes.starts_with(b"\x89PNG\r\n\x1a\n") { + ("image/png", "png") + } else if file.bytes.starts_with(&[0xFF, 0xD8, 0xFF]) { + ("image/jpeg", "jpg") + } else { + return Err("the W-9 is a PDF, PNG, or JPEG".into()); }; - Ok(ledger_vault::W9File { - file_name: canonical - .file_name() - .map(|n| n.to_string_lossy().into_owned()) - .unwrap_or_else(|| "w9".into()), - mime_type: mime_type.to_string(), - bytes, - }) + let file_name = p + .w9_file_name + .clone() + .or(file.file_name) + .unwrap_or_else(|| format!("w9.{ext}")); + if let Some(why) = ledger_store::uploads::file_name_problem(&file_name) { + return Err(why); + } + Ok(( + ledger_vault::W9File { + file_name, + mime_type: mime_type.to_string(), + bytes: file.bytes, + }, + file.claim, + )) + } + + /// A CSV's text, given inline or as an upload (books.begin_upload), + /// and the upload's hold. + fn csv_given( + &self, + text: Option<&str>, + upload_id: Option<&str>, + ) -> Result<(String, Option), String> { + match (text, upload_id) { + (Some(_), Some(_)) => Err("give csv_text or upload_id, not both".into()), + (Some(t), None) => Ok((t.to_string(), None)), + (None, Some(id)) => { + let file = self.take_file(files::Given { + upload_id: Some(id), + content_base64: None, + sha256: None, + purposes: files::IMPORTED, + })?; + let text = String::from_utf8(file.bytes) + .map_err(|_| "the CSV isn't UTF-8 text".to_string())?; + Ok((text, file.claim)) + } + (None, None) => Err("give the CSV: csv_text, or an upload_id".into()), + } } /// Record a file read or written with full TINs (tin_files.rs @@ -1920,56 +1965,6 @@ fn known_tin_last4(store: &Store) -> Vec { .collect() } -/// Where these books keep what an import must never read -/// (local_files.rs): the data dir, the books dir, the vault, the key file -/// in force and the default one, and the home folder. -fn places(store: &Store, settings: &Settings, layout: &Layout) -> Places { - Places { - data_dir: store.data_dir().to_path_buf(), - books_dir: layout.books_dir(store), - vault_dir: layout.vault_dir(store), - key_files: vec![ - layout.key_file(store, settings), - settings.key_file(), - ledger_vault::default_key_file(), - ], - // In the data root the home comes from the account's record, as - // the root's own does; never only from the environment. - home: match layout { - Layout::Root(_) => ledger_store::root::account_home().ok(), - Layout::BooksDir => None, - } - .or_else(|| std::env::var_os("HOME").map(std::path::PathBuf::from)), - } -} - -/// The import roots a server for these books may use (local_files.rs -/// INVARIANT 0): a named root that is `/`, the home folder or holds it, -/// or holds the books, the vault or a key, is refused (the server doesn't -/// start); a default one is dropped, leaving none. The text is for the -/// human starting the server, on its standard error. -pub fn guard_import_roots( - store: &Store, - settings: &Settings, - layout: &Layout, - roots: ImportRoots, -) -> Result { - let places = places(store, settings, layout); - let explicit = roots.explicit(); - let (roots, refused) = roots.guard(places.home.as_deref(), &places.keep_out()); - if let Some((root, why)) = refused.first() { - let msg = format!( - "import folder {} can't be used: {why}. Name a folder that holds none of them, such as your downloads folder", - root.display() - ); - if explicit { - return Err(msg); - } - eprintln!("ledger-mcp: {msg}; books.import_local_file has no folder to read from"); - } - Ok(roots) -} - /// Whether `path` is `dir` or inside it. fn is_inside(dir: &std::path::Path, path: &std::path::Path) -> bool { resolved(path).starts_with(resolved(dir)) @@ -2719,6 +2714,10 @@ fn movement_in(ledger: &Ledger, account: AccountId, from: NaiveDate, to: NaiveDa } thread_local! { + /// The token of the request running on this thread (books served from + /// the data root), its id and name, for what is bound to it (its + /// uploads) and the commands it is given. + static CALLER: std::cell::RefCell> = const { std::cell::RefCell::new(None) }; /// The journal's head after the last write made by the request /// running on this thread (StoreGuard's drop), for its access record. static WROTE: std::cell::RefCell> = const { std::cell::RefCell::new(None) }; @@ -2892,6 +2891,8 @@ fn lines_out(lines: &[ledger_core::StatementLine]) -> Vec { #[path = "tools/access.rs"] mod access_tools; +#[path = "tools/files.rs"] +mod files; #[path = "tools/kept.rs"] mod kept; pub use kept::SettingKeyOut; @@ -3548,7 +3549,7 @@ impl BooksServer { #[tool( name = "books.attach_receipt", title = "Attach a receipt", - description = "Attach a receipt (base64 file bytes) to an entry, or file it unlinked and link later with books.link_receipt. Files are content-addressed and deduplicated. link_to links it to more subjects in the same call (another entry it also supports, an asset, a party, ...: see books.link_attachment); every subject is checked first, so one bad subject files nothing. A file holding a full TIN is refused: one the server read or wrote with TINs, or text with a TIN the vault keeps or an SSN or EIN printed 3-2-4 or 2-7 with hyphens.", + description = "Attach a receipt to an entry, or file it unlinked and link later with books.link_receipt. The bytes: upload_id (books.begin_upload), or content_base64 with its sha256 for a file of at most 256 KiB. Files are content-addressed and deduplicated. link_to links it to more subjects in the same call (another entry it also supports, an asset, a party, ...: see books.link_attachment); every subject is checked first, so one bad subject files nothing. A file holding a full TIN is refused: one the server read or wrote with TINs, or text with a TIN the vault keeps or an SSN or EIN printed 3-2-4 or 2-7 with hyphens.", annotations( read_only_hint = false, destructive_hint = false, @@ -3560,17 +3561,6 @@ impl BooksServer { params: Parameters, ) -> Result, String> { let p = params.0; - let bytes = base64::Engine::decode( - &base64::engine::general_purpose::STANDARD, - &p.content_base64, - ) - .map_err(|_| "content_base64 is not valid base64".to_string())?; - if bytes.len() > MAX_UPLOAD_BYTES { - return Err(format!( - "file is {} bytes; the limit is {MAX_UPLOAD_BYTES}", - bytes.len() - )); - } let mut store = self.lock()?; if let Some(e) = p.entry_id && store.ledger().get_entry(e).is_none() @@ -3578,16 +3568,26 @@ impl BooksServer { return Err(format!("unknown entry {e}")); } let link_to = check_link_to(&store, p.entry_id, &p.link_to)?; - self.refuse_tin_file(&store, &bytes, &p.mime_type, &p.file_name)?; + // Taken once everything else is known good. + let file = self.take_file(files::Given { + upload_id: p.upload_id.as_deref(), + content_base64: p.content_base64.as_deref(), + sha256: p.sha256.as_deref(), + purposes: files::FILED, + })?; + let (file_name, mime_type) = named(p.file_name, p.mime_type, &file)?; + let (bytes, claim) = (file.bytes, file.claim); + self.refuse_tin_file(&store, &bytes, &mime_type, &file_name)?; let (attachment_id, sha256, deduplicated) = store .attach_receipt( p.entry_id, - &p.file_name, - &p.mime_type, + &file_name, + &mime_type, &bytes, p.description.as_deref(), ) .map_err(|e| e.to_string())?; + files::filed(claim); for sub in link_to { store .link_attachment(attachment_id, sub) @@ -3604,7 +3604,7 @@ impl BooksServer { #[tool( name = "books.file_document", title = "File a document", - description = "File a document into the content-addressed store: category formation, legal, tax, accounting, statement (a bank or card statement: then books.record_statement), contract, photo, warranty, import_source (a bank export), or other. link_to links it to subjects in the same call (an account, an asset, a party, the company, ...: see books.link_attachment); every subject is checked first, so one bad subject files nothing. Never file a W-9 here (it shows the full TIN): books.set_w9 w9_file_path puts it in the vault. A file holding a full TIN is refused: one the server read or wrote with TINs, or text with a TIN the vault keeps or an SSN or EIN printed 3-2-4 or 2-7 with hyphens.", + description = "File a document: category formation, legal, tax, accounting, statement (a bank or card statement: then books.record_statement), contract, photo, warranty, import_source (a bank export), or other. link_to links it to subjects in the same call (an account, an asset, a party, the company: see books.link_attachment), all checked first. The bytes: upload_id (books.begin_upload), or content_base64 with its sha256 (at most 256 KiB). Never a W-9 (books.set_w9 w9_upload_id puts it in the vault). A file holding a full TIN is refused: one the server read or wrote with TINs, or text with a TIN the vault keeps or an SSN or EIN printed 3-2-4 or 2-7 with hyphens.", annotations( read_only_hint = false, destructive_hint = false, @@ -3626,29 +3626,27 @@ impl BooksServer { )); } } - let bytes = base64::Engine::decode( - &base64::engine::general_purpose::STANDARD, - &p.content_base64, - ) - .map_err(|_| "content_base64 is not valid base64".to_string())?; - if bytes.len() > MAX_UPLOAD_BYTES { - return Err(format!( - "file is {} bytes; the limit is {MAX_UPLOAD_BYTES}", - bytes.len() - )); - } let mut store = self.lock()?; let link_to = check_link_to(&store, None, &p.link_to)?; - self.refuse_tin_file(&store, &bytes, &p.mime_type, &p.file_name)?; + let file = self.take_file(files::Given { + upload_id: p.upload_id.as_deref(), + content_base64: p.content_base64.as_deref(), + sha256: p.sha256.as_deref(), + purposes: files::FILED, + })?; + let (file_name, mime_type) = named(p.file_name, p.mime_type, &file)?; + let (bytes, claim) = (file.bytes, file.claim); + self.refuse_tin_file(&store, &bytes, &mime_type, &file_name)?; let (attachment_id, sha256, deduplicated) = store .file_document( &p.category, - &p.file_name, - &p.mime_type, + &file_name, + &mime_type, &bytes, p.description.as_deref(), ) .map_err(|e| e.to_string())?; + files::filed(claim); for sub in link_to { store .link_attachment(attachment_id, sub) @@ -3924,7 +3922,7 @@ impl BooksServer { #[tool( name = "books.get_receipt", title = "Get a receipt", - description = "Fetch a receipt or document's metadata (with everything it is linked to) and base64 bytes. Bytes are hash-verified on read.", + description = "A receipt or document: what it is and everything it is linked to. view bytes adds its bytes, base64 and hash-verified, for a file of at most 256 KiB; a larger one gives the command (wanderbooks download) that saves a copy in a folder you name (books served by `wanderbooks mcp`; elsewhere a larger file doesn't come back).", annotations(read_only_hint = true, open_world_hint = false) )] async fn get_receipt( @@ -3932,12 +3930,54 @@ impl BooksServer { params: Parameters, ) -> Result, String> { let p = params.0; + let bytes_wanted = match p.view.as_deref().unwrap_or("meta") { + "meta" => false, + "bytes" => true, + other => { + return Err(format!( + "view {:?}: meta (the default) or bytes", + other.chars().take(20).collect::() + )); + } + }; let store = self.lock()?; let meta = find_attachment(&store, p.attachment_id)?; - let bytes = store - .attachment_bytes(p.attachment_id) - .map_err(|e| e.to_string())?; + let book = store + .data_dir() + .file_name() + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_default(); + let fits = meta.size_bytes as usize <= files::INLINE_MAX; + let content_base64 = if bytes_wanted && fits { + let bytes = store + .attachment_bytes(p.attachment_id) + .map_err(|e| e.to_string())?; + Some(base64::Engine::encode( + &base64::engine::general_purpose::STANDARD, + &bytes, + )) + } else { + None + }; + // Too large to come back here: the command that saves a copy (books + // served from the data root; over stdio there is none). + let download = (bytes_wanted && !fits && self.uploads.is_some()).then(|| { + format!( + "wanderbooks download --book {book}{} {} --to ", + files::token_flag(), + meta.id + ) + }); + if bytes_wanted && !fits && download.is_none() { + return Err(format!( + "attachment {} is {} bytes; at most {} come back on this connection (a larger file needs these books served by `wanderbooks mcp`)", + meta.id, + meta.size_bytes, + files::INLINE_MAX + )); + } Ok(Json(GetReceiptOut { + download, links: links_out(&store, meta.id)?, description: meta.description.clone(), attachment_id: meta.id, @@ -3947,10 +3987,7 @@ impl BooksServer { file_name: meta.file_name, mime_type: meta.mime_type, size_bytes: meta.size_bytes, - content_base64: base64::Engine::encode( - &base64::engine::general_purpose::STANDARD, - &bytes, - ), + content_base64, })) } @@ -4413,7 +4450,7 @@ impl BooksServer { #[tool( name = "books.import_bank_csv", title = "Import a bank CSV as drafts", - description = "Import a QuickBooks-style bank/credit-card CSV: 3 columns Date,Description,Amount (payments negative), or 4 columns Date,Description, money out, money in (Intuit heads them Credit,Debit). Slash dates must settle month-first or day-first from the file itself or from date_order (\"month_first\" for mm/dd, \"day_first\" for dd/mm); ask the human when the tool says the dates read either way. Creates one DRAFT entry per transaction, all or none: bank account on one side, categorized offset on the other (the longest matching rule pattern, else default_offset_code). Review with books.digest, then post. File the export first (books.file_document category import_source) and pass source_attachment_id: it is linked to the bank account, so the import traces to its source (never to the drafts: a bank export is not a receipt).", + description = "Import a QuickBooks-style bank/credit-card CSV: 3 columns Date,Description,Amount (payments negative), or 4 columns Date,Description, money out, money in (Intuit heads them Credit,Debit). Slash dates must settle month-first or day-first from the file itself or from date_order (\"month_first\" for mm/dd, \"day_first\" for dd/mm); ask the human when the tool says the dates read either way. Creates one DRAFT entry per transaction, all or none: bank account on one side, categorized offset on the other (the longest matching rule pattern, else default_offset_code). Give the CSV as csv_text, or upload it (books.begin_upload purpose import) and give upload_id. Review with books.digest, then post. File the export too (books.file_document category import_source) and pass source_attachment_id: it is linked to the bank account, so the import traces to its source (never to the drafts: a bank export is not a receipt).", annotations( read_only_hint = false, destructive_hint = false, @@ -4435,8 +4472,8 @@ impl BooksServer { )); } }; - let txns = - ledger_import::parse_bank_csv_with(&p.csv_text, order).map_err(|e| e.to_string())?; + let (csv, claim) = self.csv_given(p.csv_text.as_deref(), p.upload_id.as_deref())?; + let txns = ledger_import::parse_bank_csv_with(&csv, order).map_err(|e| e.to_string())?; let mut store = self.lock()?; let bank = store .ledger() @@ -4507,6 +4544,8 @@ impl BooksServer { .map_err(|e| format!("{} {}: {e}", txn.date, txn.description)); e?; } + // Rehearsed whole: the upload is used up as the writing begins. + files::filed(claim); let mut imported = Vec::new(); for txn in txns { let (offset_id, offset_code) = offset_for(&txn.description); @@ -5578,7 +5617,7 @@ impl BooksServer { #[tool( name = "books.import_chart_csv", title = "Import a chart of accounts", - description = "Import a chart of accounts from CSV (header: code,name,type,parent_code). Types: asset, liability, equity, revenue, expense. Parents must come before children. Fails on the first bad row; nothing is half-imported.", + description = "Import a chart of accounts from CSV (header: code,name,type,parent_code): csv_text, or upload_id (books.begin_upload purpose import). Types: asset, liability, equity, revenue, expense. Parents must come before children. Fails on the first bad row; nothing is half-imported.", annotations( read_only_hint = false, destructive_hint = false, @@ -5590,7 +5629,8 @@ impl BooksServer { params: Parameters, ) -> Result, String> { let p = params.0; - let rows = ledger_import::parse_chart_csv(&p.csv_text).map_err(|e| e.to_string())?; + let (csv, claim) = self.csv_given(p.csv_text.as_deref(), p.upload_id.as_deref())?; + let rows = ledger_import::parse_chart_csv(&csv).map_err(|e| e.to_string())?; let mut store = self.lock()?; let parse_type = |t: &str| match t { "asset" => Ok(AccountType::Asset), @@ -5633,6 +5673,8 @@ impl BooksServer { .map_err(|e| format!("{}: {e}", row.code))?; } } + // Rehearsed whole: the upload is used up as the writing begins. + files::filed(claim); let mut created = Vec::new(); for row in rows { let account_type = match row.account_type.as_str() { @@ -5668,71 +5710,6 @@ impl BooksServer { Ok(Json(ImportChartCsvOut { created, count })) } - #[tool( - name = "books.import_local_file", - title = "File a local file", - description = "File a receipt or document from a file on this computer: an absolute path to a regular file in the server's import folders (by default your downloads folder; save it there first). Files of the books themselves are never read. A file the server read or wrote with full TINs (a W-9, a 1099 worksheet, a CPA package or a file from one), and text holding a TIN the vault keeps or a number printed as an SSN or EIN is (digits 3-2-4 or 2-7 joined by hyphens), are refused: they never go into the books.", - annotations( - read_only_hint = false, - destructive_hint = false, - open_world_hint = false - ) - )] - async fn import_local_file( - &self, - params: Parameters, - ) -> Result, String> { - let p = params.0; - // The name and type are those of the file read, where the path - // really leads (a link named receipt.pdf to a CSV is a CSV). - let (canonical, bytes) = self.read_local(&p.path)?; - let file_name = p.file_name.unwrap_or_else(|| { - canonical - .file_name() - .and_then(|n| n.to_str()) - .unwrap_or("file") - .to_string() - }); - let mime_type = mime_guess::from_path(&canonical) - .first_or_octet_stream() - .to_string(); - let mut store = self.lock()?; - // A file the server read or wrote with full TINs, or text holding - // one (the vault's, or one printed as an SSN or EIN is), never goes - // into the books (tin_files.rs). - self.refuse_tin_file(&store, &bytes, &mime_type, &file_name)?; - let (attachment_id, sha256, deduplicated) = match p.category.as_str() { - "receipt" => store - .attach_receipt( - p.entry_id, - &file_name, - &mime_type, - &bytes, - p.description.as_deref(), - ) - .map_err(|e| e.to_string())?, - "document" => { - let cat = p.document_category.as_deref().unwrap_or("other"); - store - .file_document( - cat, - &file_name, - &mime_type, - &bytes, - p.description.as_deref(), - ) - .map_err(|e| e.to_string())? - } - _ => return Err("category must be \"receipt\" or \"document\"".to_string()), - }; - Ok(Json(ImportLocalFileOut { - attachment_id, - sha256, - deduplicated, - file_name, - })) - } - #[tool( name = "books.export", title = "Export a report", @@ -7097,7 +7074,7 @@ impl BooksServer { #[tool( name = "books.set_w9", title = "Record W-9 facts", - description = "Record what a contractor's or landlord's W-9 says, or clear it (clear: true alone): the date received (received_on), line 3 entity type, line 4 exempt payee and FATCA codes, the TIN's last four digits, and where the form is kept. Give the full TIN as tin with tin_type (ssn, ein, itin) and it goes only to the encrypted TIN vault (its key kept outside the books, created on first use): the books keep its last four (tin_last4 is then optional, and refused if it disagrees) and a hash of the vault record, in the W-9 facts and the party's TIN facts alike (party.tin_set, as books.set_tin), and kept_at defaults to \"wanderbooks vault\". The CPA package carries the full TIN to the CPA. Give w9_file_path (a local PDF, PNG, or JPEG of the signed form) and the form goes to the vault too, never to the receipts and documents; read it back with books.get_w9. Never put a TIN in any other field, note, description, or file: the engine refuses any nine-digit number, and no tool output echoes one. The party needs a 1099 role (contractor or landlord: books.set_party_roles). Record it before paying them. A party with a vendor record also gets its W-9 marked on file there (books.vendor_1099 missing_w9). Clearing W-9 facts whose TIN is in the vault needs books.purge_tin first. Journaled as party.w9_set. Returns the party (the TIN's last four only).", + description = "Record what a contractor's or landlord's W-9 says, or clear it (clear: true alone): the date received (received_on), line 3 entity type, line 4 exempt payee and FATCA codes, the TIN's last four digits, and where the form is kept. Give the full TIN as tin with tin_type (ssn, ein, itin) and it goes only to the encrypted TIN vault (its key kept outside the books, created on first use): the books keep its last four (tin_last4 is then optional, and refused if it disagrees) and a hash of the vault record, in the W-9 facts and the party's TIN facts alike (party.tin_set, as books.set_tin), and kept_at defaults to \"wanderbooks vault\". The CPA package carries the full TIN to the CPA. Give the signed form (a PDF, PNG, or JPEG) as w9_upload_id (books.begin_upload), or w9_content_base64 with w9_sha256 when at most 256 KiB, and it goes to the vault too, never to the receipts and documents; read it back with books.get_w9. Never put a TIN in any other field, note, description, or file: the engine refuses any nine-digit number, and no tool output echoes one. The party needs a 1099 role (contractor or landlord: books.set_party_roles). Record it before paying them. A party with a vendor record also gets its W-9 marked on file there (books.vendor_1099 missing_w9). Clearing W-9 facts whose TIN is in the vault needs books.purge_tin first. Journaled as party.w9_set. Returns the party (the TIN's last four only).", annotations( read_only_hint = false, destructive_hint = true, @@ -7106,7 +7083,14 @@ impl BooksServer { )] async fn set_w9(&self, params: Parameters) -> Result, String> { let p = params.0; - let vault_input = p.tin.is_some() || p.tin_type.is_some() || p.w9_file_path.is_some(); + let w9_given = p.w9_upload_id.is_some() || p.w9_content_base64.is_some(); + if p.w9_file_name.is_some() && !w9_given { + return Err( + "w9_file_name names the W-9 given with it (w9_upload_id or w9_content_base64)" + .into(), + ); + } + let vault_input = p.tin.is_some() || p.tin_type.is_some() || w9_given; let tin = match (&p.tin, &p.tin_type) { (Some(t), Some(k)) => Some( ledger_vault::Tin::new(t.0.trim(), parse_tin_type(k)?) @@ -7159,11 +7143,12 @@ impl BooksServer { vault_record: None, }) }; - let w9_file = p - .w9_file_path - .as_deref() - .map(|path| self.read_w9_file(path)) - .transpose()?; + let (w9_file, mut w9_claim) = if w9_given { + let (file, claim) = self.w9_file(&p)?; + (Some(file), claim) + } else { + (None, None) + }; let mut store = self.lock()?; let id = resolve_party(&store, &p.party)?; let existing = store.ledger().get_party(id).and_then(|p| p.w9.clone()); @@ -7245,6 +7230,8 @@ impl BooksServer { Some(w) => Some(vault.put_w9(id, w).map_err(|e| e.to_string())?), None => None, }; + // The W-9 is in the vault: its upload is used up. + files::filed(w9_claim.take()); match tin_hash { Some(h) => Some(h), None if held_tin => vault.record_hash(id).map_err(|e| e.to_string())?, @@ -7423,7 +7410,7 @@ impl BooksServer { .map_err(|e| e.to_string())? .ok_or_else(|| { format!( - "the TIN vault holds no W-9 file for party {id}: store one with books.set_w9 w9_file_path" + "the TIN vault holds no W-9 file for party {id}: store one with books.set_w9 w9_upload_id" ) })?; self.wrote_tins(&store, &file.bytes)?; @@ -8500,11 +8487,6 @@ impl BooksServer { ) }) .transpose()?; - if p.include_tins - && let Some(d) = &dest - { - self.outside_import_roots(d)?; - } let meta = self.report_meta(&store); let ledger = store.ledger(); // The vault, opened once and never creating a key: locked, each @@ -9588,7 +9570,10 @@ impl BooksServer { .parse() .map_err(|_| format!("bad receipt resource URI: {uri}"))?; let Json(out) = self - .get_receipt(Parameters(GetReceiptParams { attachment_id: id })) + .get_receipt(Parameters(GetReceiptParams { + attachment_id: id, + view: None, + })) .await?; serde_json::to_string_pretty(&out).map_err(|e| e.to_string()) } @@ -9703,6 +9688,11 @@ impl ServerHandler for BooksServer { let result = run_blocking(&self.store, move || { Box::pin(async move { WROTE.with(|w| w.borrow_mut().take()); + let caller = running_call.as_ref().and_then(|c| { + c.caller() + .map(|c| (c.token.id.clone(), c.token.name.clone())) + }); + CALLER.with(|c| *c.borrow_mut() = caller); // Counted as running by now (run_blocking): hashed here. let digest = running_call .as_ref() @@ -10073,11 +10063,16 @@ mod tests { } } - /// "Max 25 MB" (books.attach_receipt): 25 MiB exactly. + /// An upload's limit, 25 MiB exactly; inline, 256 KiB (plan 2.1, 4.7). #[test] - fn the_upload_limit_is_25_mib() { - assert_eq!(MAX_UPLOAD_BYTES, 25 * 1024 * 1024); - assert_eq!(MAX_UPLOAD_BYTES, 26_214_400); + fn the_file_limits_are_the_plans() { + assert_eq!(ledger_store::uploads::MAX_UPLOAD, 26_214_400); + assert_eq!(files::INLINE_MAX, 262_144); + // Plan 4.7: an upload lasts 15 minutes; 16 under way per token. + assert_eq!(ledger_store::uploads::UPLOAD_TTL, 900); + assert_eq!(ledger_store::uploads::MAX_LIVE, 16); + assert_eq!(ledger_store::uploads::MAX_NAME_BYTES, 255); + assert_eq!(crate::local::MAX_BODY, 1024 * 1024); } /// A period is "YYYY-MM": a year of the calendar the books keep @@ -10450,7 +10445,7 @@ mod tests { ("books.assign_role", D), ("books.set_tax_category", W), ("books.import_chart_csv", W), - ("books.import_local_file", W), + ("books.begin_upload", W), ("books.export", R), ("books.register_asset", W), ("books.asset_schedule", R), @@ -10577,6 +10572,10 @@ mod tests { "ledger-import xlsx", include_str!("../../ledger-import/src/xlsx.rs"), ), + ( + "ledger-store uploads", + include_str!("../../ledger-store/src/uploads.rs"), + ), ] { let mut rest = src; let mut n = 0; @@ -10589,6 +10588,18 @@ mod tests { } assert!(n > 0, "{krate}: no error templates found"); } + // The files tool's refusals (and description): its sentences. + for line in include_str!("tools/files.rs").lines() { + let line = line.trim_start(); + if line.starts_with("//") { + continue; + } + for (i, text) in line.split('"').enumerate() { + if i % 2 == 1 && text.len() > 30 && text.contains(' ') { + out.push(("tools/files.rs text".into(), text.to_string())); + } + } + } let newer = |verified| ledger_store::StoreError::NewerJournal { seq: 7, event_type: "settings.set".into(), @@ -10605,25 +10616,11 @@ mod tests { )); for (what, text) in [ ("STOPPED", STOPPED), - ("TIN_FILE_IN_ROOT", TIN_FILE_IN_ROOT), ("KEY_IN_BOOKS", KEY_IN_BOOKS), ("VAULT_IN_HISTORY", VAULT_IN_HISTORY), ("INSIDE_BOOKS", INSIDE_BOOKS), ("NO_FOLDER", NO_FOLDER), ("EXISTS", EXISTS), - ("local_files::CHANGED", crate::local_files::CHANGED), - ( - "local_files::NO_SUCH_FILE", - crate::local_files::NO_SUCH_FILE, - ), - ("local_files::NO_ROOTS", crate::local_files::NO_ROOTS), - ("local_files::SEALED", crate::local_files::SEALED), - ( - "local_files::IN_THE_BOOKS", - crate::local_files::IN_THE_BOOKS, - ), - ("local_files::HARD_LINKED", crate::local_files::HARD_LINKED), - ("local_files::NOT_REGULAR", crate::local_files::NOT_REGULAR), ("tin_files::TIN_FILE", tin_files::TIN_FILE), ("tin_files::TIN_HELD", tin_files::TIN_HELD), ("tin_files::TIN_PROBE", tin_files::TIN_PROBE), @@ -10768,6 +10765,69 @@ mod tests { ); } + /// No tool takes a path on the server's computer (plan 4.7, D-28): + /// files come by upload or inline. The three outputs with full TINs + /// keep theirs until the viewer takes them over (P2 slice 6). + #[test] + fn no_tool_takes_a_path() { + const UNTIL_THE_VIEWER: &[(&str, &str)] = &[ + ("books.export_cpa_package", "save_to"), + ("books.vendor_1099", "save_csv_to"), + ("books.get_w9", "save_to"), + ]; + fn names(v: &serde_json::Value, out: &mut Vec) { + match v { + serde_json::Value::Object(map) => { + if let Some(serde_json::Value::Object(props)) = map.get("properties") { + out.extend(props.keys().cloned()); + } + map.values().for_each(|v| names(v, out)); + } + serde_json::Value::Array(items) => items.iter().for_each(|v| names(v, out)), + _ => {} + } + } + let mut found = Vec::new(); + for t in BooksServer::tool_router().list_all() { + let mut props = Vec::new(); + names( + &serde_json::Value::Object((*t.input_schema).clone()), + &mut props, + ); + for p in props { + let lower = p.to_ascii_lowercase(); + // Names that hold a place to read or write: whole words, + // so `link_to`, `*_file_name`, `tin_on_file` and `date_to` + // pass. + let words: Vec<&str> = lower.split('_').collect(); + let pathlike = lower.starts_with("save") + || lower.contains("path") + || [ + "dir", + "folder", + "root", + "dest", + "destination", + "output", + "location", + "local", + ] + .iter() + .any(|w| words.contains(w)) + || (words.last() == Some(&"file") && !lower.ends_with("on_file")) + || lower == "to" + || lower == "from" + || lower == "source"; + if pathlike && !UNTIL_THE_VIEWER.contains(&(&*t.name, p.as_str())) { + found.push(format!("{} {p}", t.name)); + } + } + } + found.sort(); + found.dedup(); + assert!(found.is_empty(), "path-like parameters: {found:?}"); + } + #[test] fn tool_text_lint() { const NEVER: &[&str] = &[ @@ -10793,6 +10853,13 @@ mod tests { "read the file", "open the file", "cat ", + // Gone in P2 slice 5: files come by upload or inline. + "import_local_file", + "w9_file_path", + "import folder", + "import root", + "put_url", + "human_url", "fl-0001", ]; // Whole words only: "digit" and "committee" are fine. @@ -10830,6 +10897,45 @@ mod tests { )); } } + // Every tool the text names is one there is. + let tools: std::collections::HashSet = BooksServer::tool_router() + .list_all() + .into_iter() + .map(|t| t.name.to_string()) + .collect(); + let mut unknown: Vec = Vec::new(); + // (Settings keys, `books.cash_accounts`, are named like tools.) + for (what, text) in texts + .iter() + .filter(|(w, _)| !w.contains("settings literal")) + { + for (i, _) in text.match_indices("books.") { + let name: String = text[i..] + .chars() + .take_while(|c| { + c.is_ascii_lowercase() || c.is_ascii_digit() || *c == '_' || *c == '.' + }) + .collect(); + let name = name.trim_end_matches('.'); + // "the books." ends a sentence. + if name == "books" { + continue; + } + let before = text[..i].chars().next_back(); + if before.is_some_and(|c| c.is_alphanumeric() || c == '_' || c == '/') { + continue; + } + if !tools.contains(name) { + unknown.push(format!("{what}: {name}")); + } + } + } + unknown.sort(); + unknown.dedup(); + assert!( + unknown.is_empty(), + "text names tools there aren't: {unknown:?}" + ); let words = |text: &str| { text.split(|c: char| !c.is_alphanumeric() && c != '_') .map(str::to_string) diff --git a/crates/ledger-mcp/src/tools/files.rs b/crates/ledger-mcp/src/tools/files.rs new file mode 100644 index 0000000..4704dfb --- /dev/null +++ b/crates/ledger-mcp/src/tools/files.rs @@ -0,0 +1,266 @@ +//! Files in (plan 4.7, P2 slice 5, D-28): `books.begin_upload`, and the +//! bytes a tool is given, as an upload or inline. +//! +//! ```text +//! GOAL: an agent files a document of any size up to the limit without +//! its bytes passing through its context, and a small one inline, +//! both exactly as the agent declared them. +//! SOURCE: plan 4.7 (begin_upload; "content_base64 (≤256 KiB + required +//! sha256)"; guard base64_corrupt), 2.1 ("256 KiB with a mandatory +//! matching sha256"), D-28 (the agent does it all). +//! INVARIANTS: +//! 1. A tool's bytes are an upload made by the same token, or inline +//! base64 of at most INLINE_MAX bytes whose SHA-256 is the one given; +//! never both, never neither. An upload is used up only by the call +//! that files it (TakenFile::filed); a call that fails leaves it for +//! the next. +//! 2. begin_upload makes nothing for a receipt or document whose bytes +//! are stored already: it names the attachment (not withdrawn) that +//! holds them. +//! 3. Uploads exist only for books served from the data root (the +//! server's socket carries the bytes); over stdio, only inline. +//! 4. An upload is filed only by a tool of the purpose it was begun for +//! (a receipt or document, a CSV import, a W-9); another tool's +//! refusal leaves it for the right one. +//! 5. Inline text longer than INLINE_MAX bytes' base64 is refused +//! before it is decoded. +//! ``` + +use super::*; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +/// The most bytes given inline (plan 2.1: 256 KiB). +pub(crate) const INLINE_MAX: usize = 256 * 1024; + +/// Why an upload is refused on a connection that has none (INVARIANT 3). +const STDIO_ONLY_INLINE: &str = "this connection takes files inline only (content_base64 with its sha256, at most 256 KiB): a larger one needs these books served by `wanderbooks mcp`; tell the human"; + +/// What the bytes of a file a tool files came as (INVARIANT 1), and the +/// upload purposes the tool takes (INVARIANT 4). +pub(crate) struct Given<'a> { + pub upload_id: Option<&'a str>, + pub content_base64: Option<&'a str>, + pub sha256: Option<&'a str>, + pub purposes: &'a [&'a str], +} + +/// What attach_receipt and file_document take. +pub(crate) const FILED: &[&str] = &["receipt", "document"]; +/// What the CSV imports take. +pub(crate) const IMPORTED: &[&str] = &["import"]; +/// What set_w9 takes. +pub(crate) const W9: &[&str] = &["w9"]; + +/// The bytes, and the name and type an upload declared. +pub(crate) struct TakenFile { + pub bytes: Vec, + pub file_name: Option, + pub mime: Option, + /// The upload's hold, if it came as one (INVARIANT 1). + pub claim: Option, +} + +/// ` --token NAME` for a command given to the agent, when its token isn't +/// the one the commands use by default: the command runs as the caller. +pub(crate) fn token_flag() -> String { + CALLER.with(|c| match c.borrow().as_ref() { + Some((_, name)) if name != ledger_store::auth::DEFAULT_NAME => { + format!(" --token {name}") + } + _ => String::new(), + }) +} + +/// The bytes are in the books: an upload they came as is used up +/// (INVARIANT 1). Dropped instead, it is given back. +pub(crate) fn filed(claim: Option) { + if let Some(c) = claim { + c.spend(); + } +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct BeginUploadParams { + /// The file's SHA-256, hex (`sha256sum FILE`). + pub sha256: String, + /// Its size in bytes (at most 25 MiB). + pub size: u64, + /// Its MIME type, e.g. "application/pdf". + pub mime_type: String, + /// Its name, e.g. "receipt.pdf". + pub file_name: String, + /// What it is for: receipt, document, import (a bank or chart CSV), or + /// w9 (books.set_w9). + pub purpose: String, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct BeginUploadOut { + /// The upload's id: name it in the tool that files it. + #[serde(skip_serializing_if = "Option::is_none")] + pub upload_id: Option, + /// The command that sends the file (run it with the file's path). + #[serde(skip_serializing_if = "Option::is_none")] + pub command: Option, + /// When the upload lapses (UTC). + #[serde(skip_serializing_if = "Option::is_none")] + pub expires_at: Option, + /// The most bytes taken. + pub max_bytes: u64, + /// These bytes are stored already, in this attachment: link it + /// (books.link_attachment) instead of filing a copy. + #[serde(skip_serializing_if = "Option::is_none")] + pub existing_attachment_id: Option, +} + +impl BooksServer { + /// The token of the call running on this thread (its id), if any. + fn caller_token(&self) -> Option { + CALLER.with(|c| c.borrow().as_ref().map(|(id, _)| id.clone())) + } + + /// The bytes a tool was given (INVARIANT 1). + pub(crate) fn take_file(&self, given: Given<'_>) -> Result { + match (given.upload_id, given.content_base64) { + (Some(_), Some(_)) => Err("give upload_id or content_base64, not both".into()), + (None, None) => Err( + "the file's bytes are missing: upload it (books.begin_upload), or give content_base64 with its sha256 (at most 256 KiB)" + .into(), + ), + (Some(id), None) => { + // INVARIANT 3. + let (Some(uploads), Some(token)) = (&self.uploads, self.caller_token()) else { + return Err(STDIO_ONLY_INLINE.into()); + }; + let got = uploads + .take(id.trim(), &token, chrono::Utc::now().timestamp()) + .map_err(|e| e.to_string())?; + // INVARIANT 4: dropped, the claim gives it back. + if !given.purposes.contains(&got.grant.purpose.as_str()) { + return Err(format!( + "that upload was begun for {}: this tool takes one begun for {}", + got.grant.purpose, + given.purposes.join(" or ") + )); + } + Ok(TakenFile { + bytes: got.bytes, + file_name: Some(got.grant.file_name), + mime: Some(got.grant.mime), + claim: Some(got.claim), + }) + } + (None, Some(b64)) => { + let Some(sha) = given.sha256 else { + return Err("content_base64 comes with its sha256 (the file's SHA-256, hex), so a byte lost on the way is caught".into()); + }; + // INVARIANT 5. + if b64.len() > INLINE_MAX.div_ceil(3) * 4 { + return Err(format!( + "inline is at most {INLINE_MAX} bytes: upload it with books.begin_upload" + )); + } + let bytes = base64::Engine::decode(&base64::engine::general_purpose::STANDARD, b64) + .map_err(|_| "content_base64 is not valid base64 (guard base64_corrupt): send the file with books.begin_upload".to_string())?; + if bytes.len() > INLINE_MAX { + return Err(format!( + "the file is {} bytes; inline is at most {INLINE_MAX}: upload it with books.begin_upload", + bytes.len() + )); + } + if ledger_store::journal::sha256_hex(&bytes) != sha.trim().to_ascii_lowercase() { + return Err("content_base64 doesn't have the sha256 given: some of it was lost on the way (guard base64_corrupt). Send the file with books.begin_upload instead".into()); + } + Ok(TakenFile { + bytes, + file_name: None, + mime: None, + claim: None, + }) + } + } + } +} + +#[tool_router(router = files_router, vis = "pub(super)")] +impl BooksServer { + #[tool( + name = "books.begin_upload", + title = "Begin an upload", + description = "Send a file of up to 25 MiB (books served by `wanderbooks mcp`; elsewhere only inline) without its bytes passing through you: give its sha256 (sha256sum), size, type, name and purpose; run the command returned with the file's path; then name the upload_id in books.attach_receipt, books.file_document, books.set_w9, books.import_bank_csv or books.import_chart_csv. Purpose: receipt or document (those two tools), import (the CSV imports), w9. If a receipt's or document's bytes are stored already, existing_attachment_id names them: link that instead. An upload lasts 15 minutes and is used once.", + annotations( + read_only_hint = false, + destructive_hint = false, + open_world_hint = false + ) + )] + async fn begin_upload( + &self, + params: Parameters, + ) -> Result, String> { + let p = params.0; + let max = ledger_store::uploads::MAX_UPLOAD; + let sha = p.sha256.trim().to_ascii_lowercase(); + if !ledger_store::uploads::PURPOSES.contains(&p.purpose.as_str()) { + return Err("purpose is receipt, document, import (a bank or chart CSV) or w9".into()); + } + // INVARIANT 2: an attachment that can be linked, for a file that + // would be one (an import or a W-9 needs its bytes). + let store = self.lock()?; + let existing = if FILED.contains(&p.purpose.as_str()) { + store + .list_attachments(None) + .map_err(|e| e.to_string())? + .into_iter() + .find(|a| a.blob_sha == sha && a.withdrawn_at.is_none()) + .map(|a| a.id) + } else { + None + }; + let book = store + .data_dir() + .file_name() + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_default(); + drop(store); + if let Some(id) = existing { + return Ok(Json(BeginUploadOut { + upload_id: None, + command: None, + expires_at: None, + max_bytes: max, + existing_attachment_id: Some(id), + })); + } + let (Some(uploads), Some(token)) = (&self.uploads, self.caller_token()) else { + return Err(STDIO_ONLY_INLINE.into()); + }; + let now = chrono::Utc::now(); + let grant = uploads + .grant( + &token, + &sha, + p.size, + &p.mime_type, + &p.file_name, + &p.purpose, + now.timestamp(), + ) + .map_err(|e| e.to_string())?; + let expires = chrono::DateTime::from_timestamp(grant.expires, 0) + .map(|t| t.to_rfc3339_opts(chrono::SecondsFormat::Secs, true)) + .unwrap_or_default(); + Ok(Json(BeginUploadOut { + command: Some(format!( + "wanderbooks upload --book {book}{} {} -- ", + token_flag(), + grant.id + )), + upload_id: Some(grant.id), + expires_at: Some(expires), + max_bytes: max, + existing_attachment_id: None, + })) + } +} diff --git a/crates/ledger-mcp/src/types.rs b/crates/ledger-mcp/src/types.rs index d5dcebb..c0ddf22 100644 --- a/crates/ledger-mcp/src/types.rs +++ b/crates/ledger-mcp/src/types.rs @@ -369,7 +369,7 @@ pub struct Vendor1099Params { pub include_tins: bool, /// Also write the worksheet as CSV to this local path (TINs included /// only with include_tins). Never inside the books dir, and with TINs - /// never in the server's import folders; an existing file is refused. + /// an existing file is refused. pub save_csv_to: Option, } @@ -868,8 +868,12 @@ pub struct CashFlowOut { #[derive(Debug, Deserialize, JsonSchema)] pub struct ImportBankCsvParams { /// The CSV text (QBO bank upload layout: Date,Description,Amount, or - /// Date,Description, money out, money in). - pub csv_text: String, + /// Date,Description, money out, money in)… + #[serde(default)] + pub csv_text: Option, + /// …or an upload of it made with books.begin_upload (purpose import). + #[serde(default)] + pub upload_id: Option, /// Account code for the bank account (the offset side of each entry). pub bank_account_code: String, /// Account code for the default offset (e.g. an expense account). @@ -923,8 +927,12 @@ pub struct ImportBankCsvOut { /// Parameters for `books.import_chart_csv`. #[derive(Debug, Deserialize, JsonSchema)] pub struct ImportChartCsvParams { - /// The CSV text: code,name,type,parent_code with a header row. - pub csv_text: String, + /// The CSV text: code,name,type,parent_code with a header row… + #[serde(default)] + pub csv_text: Option, + /// …or an upload of it made with books.begin_upload (purpose import). + #[serde(default)] + pub upload_id: Option, } /// Result of `books.import_chart_csv`. @@ -936,41 +944,6 @@ pub struct ImportChartCsvOut { pub count: usize, } -/// Parameters for `books.import_local_file`. -#[derive(Debug, Deserialize, JsonSchema)] -pub struct ImportLocalFileParams { - /// Absolute path to a regular file in the server's import folders (by - /// default your downloads folder). - pub path: String, - /// "receipt" or "document". - pub category: String, - /// Entry id to attach to (receipts only). Omit to file unlinked. - #[serde(default)] - pub entry_id: Option, - /// Document category for documents: formation, legal, tax, accounting, other. - #[serde(default)] - pub document_category: Option, - /// Override the file name (defaults to the path's file name). - #[serde(default)] - pub file_name: Option, - /// Optional description. - #[serde(default)] - pub description: Option, -} - -/// Result of `books.import_local_file`. -#[derive(Debug, Serialize, JsonSchema)] -pub struct ImportLocalFileOut { - /// Attachment id. - pub attachment_id: u64, - /// SHA-256 hex of the content. - pub sha256: String, - /// True if the content was already stored (deduplicated). - pub deduplicated: bool, - /// File name used. - pub file_name: String, -} - /// Parameters for `books.balance_sheet`. #[derive(Debug, Deserialize, JsonSchema)] pub struct BalanceSheetParams { @@ -1270,12 +1243,23 @@ pub struct AttachReceiptParams { /// Entry id to attach to. Omit to file unlinked, then link later with /// `books.link_receipt`. pub entry_id: Option, - /// Original file name, e.g. "receipt.png". - pub file_name: String, - /// MIME type, e.g. "image/png". - pub mime_type: String, - /// File bytes, base64-encoded. Max 25 MB. - pub content_base64: String, + /// Original file name, e.g. "receipt.png" (an upload's own, if not + /// given). + #[serde(default)] + pub file_name: Option, + /// MIME type, e.g. "image/png" (an upload's own, if not given). + #[serde(default)] + pub mime_type: Option, + /// The file's bytes: an upload made with books.begin_upload (its id)… + #[serde(default)] + pub upload_id: Option, + /// …or, for a file of at most 256 KiB, its bytes base64-encoded, with + /// sha256. + #[serde(default)] + pub content_base64: Option, + /// The SHA-256 of the bytes in content_base64, hex. + #[serde(default)] + pub sha256: Option, /// Optional description. pub description: Option, /// More subjects to link it to in the same call (each journaled as its @@ -1304,12 +1288,22 @@ pub struct FileDocumentParams { /// One of: formation, legal, tax, accounting, statement, contract, /// photo, warranty, import_source, other. pub category: String, - /// Original file name. - pub file_name: String, - /// MIME type. - pub mime_type: String, - /// File bytes, base64-encoded. Max 25 MB. - pub content_base64: String, + /// Original file name (an upload's own, if not given). + #[serde(default)] + pub file_name: Option, + /// MIME type (an upload's own, if not given). + #[serde(default)] + pub mime_type: Option, + /// The file's bytes: an upload made with books.begin_upload (its id)… + #[serde(default)] + pub upload_id: Option, + /// …or, for a file of at most 256 KiB, its bytes base64-encoded, with + /// sha256. + #[serde(default)] + pub content_base64: Option, + /// The SHA-256 of the bytes in content_base64, hex. + #[serde(default)] + pub sha256: Option, /// Optional description. pub description: Option, /// Subjects to link it to in the same call (each journaled as its own @@ -1421,10 +1415,16 @@ pub struct ListReceiptsOut { pub struct GetReceiptParams { /// Attachment id. pub attachment_id: u64, + /// What to return: "meta" (default: what it is and what it's linked + /// to), or "bytes" (also its bytes, base64, for a file of at most + /// 256 KiB; a larger one comes with `wanderbooks download`). + #[serde(default)] + pub view: Option, } -/// Result of `books.get_receipt`: metadata plus base64 bytes. The bytes are -/// hash-verified on read. +/// Result of `books.get_receipt`: what it is and what it's linked to, and +/// with view `bytes`, its bytes (hash-verified on read) or the command that +/// saves a copy. #[derive(Debug, Serialize, JsonSchema)] pub struct GetReceiptOut { /// Attachment id. @@ -1445,8 +1445,14 @@ pub struct GetReceiptOut { pub description: Option, /// Everything it is linked to (the entry it is filed on first). pub links: Vec, - /// File bytes, base64-encoded. - pub content_base64: String, + /// Its bytes, base64-encoded: with view `bytes`, for a file of at most + /// 256 KiB. + #[serde(skip_serializing_if = "Option::is_none")] + pub content_base64: Option, + /// With view `bytes`, for a file too large to come back here: the + /// command that saves a copy in a folder you name. + #[serde(skip_serializing_if = "Option::is_none")] + pub download: Option, } // --------------------------------------------------------------------------- @@ -2605,8 +2611,7 @@ pub struct ExportCpaPackageParams { pub include_tins: bool, /// Where to write the package: a new `.zip` file, or an existing /// folder (the zip goes in it as `cpa-package-.zip`). Written - /// readable by its owner only. With TINs, never inside the books dir - /// or the server's import folders. + /// readable by its owner only. With TINs, never inside the books dir. pub save_to: Option, } @@ -2727,11 +2732,22 @@ pub struct SetW9Params { pub tin: Option, /// The TIN's type: ssn, ein, or itin. Required with tin. pub tin_type: Option, - /// A local PDF, PNG, or JPEG of the signed W-9, in the server's import - /// folders (by default your downloads folder): stored only in the - /// vault (it shows the full TIN), never with the receipts and - /// documents. Read it back with `books.get_w9`. - pub w9_file_path: Option, + /// The signed W-9 itself (a PDF, PNG, or JPEG): an upload made with + /// books.begin_upload. Stored only in the vault (it shows the full + /// TIN), never with the receipts and documents. + #[serde(default)] + pub w9_upload_id: Option, + /// …or, for one of at most 256 KiB, its bytes base64-encoded, with + /// w9_sha256. + #[serde(default)] + pub w9_content_base64: Option, + /// The SHA-256 of the bytes in w9_content_base64, hex. + #[serde(default)] + pub w9_sha256: Option, + /// The W-9 file's name (default: the upload's, else w9.pdf, w9.png + /// or w9.jpg by its type). + #[serde(default)] + pub w9_file_name: Option, } /// A full TIN as a tool takes it: text (or a JSON number) whose value is @@ -2807,8 +2823,7 @@ pub struct GetW9Params { /// The party: id or name. pub party: String, /// Where to write the W-9 file: a new local file outside the books - /// and outside the server's import folders (the file shows the full - /// TIN). An existing file is refused. + /// (the file shows the full TIN). An existing file is refused. pub save_to: String, } diff --git a/crates/ledger-mcp/tests/hostile_args.rs b/crates/ledger-mcp/tests/hostile_args.rs index 8971f9a..7b1556d 100644 --- a/crates/ledger-mcp/tests/hostile_args.rs +++ b/crates/ledger-mcp/tests/hostile_args.rs @@ -67,8 +67,9 @@ impl Drop for Mcp { } } -/// The places one run uses: the books, the vault key, the import root, the -/// folder the server runs in (where a relative path lands). +/// The places one run uses: the books, the vault key, a folder of files on +/// their way in (as a downloads folder), the folder the server runs in +/// (where a relative path lands). struct Places { base: PathBuf, books: PathBuf, @@ -130,8 +131,6 @@ impl Mcp { .arg(p.books.join("books-data")) .arg("--books-toml") .arg(p.books.join("wanderbooks.toml")) - .arg("--import-root") - .arg(&p.inbox) .current_dir(&p.sandbox) .env("HOME", p.base.join("home")) .env("XDG_CONFIG_HOME", p.base.join("config")) @@ -256,7 +255,8 @@ fn fixture(m: &mut Mcp, p: &Places) { m.ok( "books.attach_receipt", json!({"file_name": "r.pdf", "mime_type": "application/pdf", - "content_base64": "JVBERi0xLjQgYSByZWNlaXB0"}), + "content_base64": "JVBERi0xLjQgYSByZWNlaXB0", + "sha256": ledger_store::journal::sha256_hex(b"%PDF-1.4 a receipt")}), ); for (date, dr, cr, amount) in [ ("2026-01-02", "1000", "3000", "5000.00"), @@ -279,7 +279,8 @@ fn fixture(m: &mut Mcp, p: &Places) { m.ok( "books.attach_receipt", json!({"entry_id": id, "file_name": "r.pdf", "mime_type": "application/pdf", - "content_base64": b64(pdf.as_bytes())}), + "content_base64": b64(pdf.as_bytes()), + "sha256": ledger_store::journal::sha256_hex(pdf.as_bytes())}), ); m.ok( "books.post_entry", @@ -321,7 +322,8 @@ fn fixture(m: &mut Mcp, p: &Places) { m.ok( "books.file_document", json!({"category": "statement", "file_name": "s.pdf", "mime_type": "application/pdf", - "content_base64": "JVBERi0xLjQgYSBzdGF0ZW1lbnQ="}), + "content_base64": "JVBERi0xLjQgYSBzdGF0ZW1lbnQ=", + "sha256": ledger_store::journal::sha256_hex(b"%PDF-1.4 a statement")}), ); let _ = p; } @@ -500,8 +502,8 @@ fn hostile_numbers() -> Vec { ] } -/// Paths a tool may be given: inside the sandbox, the import root, the -/// books, relative ones, odd ones. Never anywhere a real file lives. +/// Paths a tool may be given: inside the sandbox, the inbox, the books, +/// relative ones, odd ones. Never anywhere a real file lives. fn hostile_paths(p: &Places) -> Vec { let s = |p: &Path| p.to_str().unwrap().to_string(); vec![ @@ -618,17 +620,20 @@ fn string_for(name: &str, node: &Value, p: &Places) -> BoxedStrategy { .boxed(); } if n.contains("base64") { - likely.extend( - [ - "JVBERi0xLjQ=", - "", - "!!!not base64", - "AAAA", - "NTcxLTkzLTI4NDY=", - "RGF0ZSxBbW91bnQKMjAyNi0wMS0wMSwxCg==", - ] - .map(String::from), - ); + likely.extend(INLINE_FILES.map(String::from)); + } + // A file's SHA-256: of one of the files above (so some calls carry a + // matching pair and go on past the check), or not. + if n.contains("sha256") { + use base64::Engine; + likely.extend(INLINE_FILES.iter().filter_map(|f| { + base64::engine::general_purpose::STANDARD + .decode(f) + .ok() + .map(|b| ledger_store::journal::sha256_hex(&b)) + })); + likely.push("0".repeat(64)); + likely.push("not hex".into()); } let pick = |pred: fn(&str) -> bool| -> Vec { hostile_strings().into_iter().filter(|s| pred(s)).collect() @@ -694,6 +699,16 @@ fn string_for(name: &str, node: &Value, p: &Places) -> BoxedStrategy { .boxed() } +/// Files given inline (base64), hostile and not. +const INLINE_FILES: [&str; 6] = [ + "JVBERi0xLjQ=", + "", + "!!!not base64", + "AAAA", + "NTcxLTkzLTI4NDY=", + "RGF0ZSxBbW91bnQKMjAyNi0wMS0wMSwxCg==", +]; + /// An object's value: each property present or not (required ones mostly /// present), and now and then a field no schema names. fn object_for( @@ -1144,7 +1159,8 @@ fn inputs_that_once_panicked_are_refused_or_answered() { &mut m, "books.attach_receipt", json!({"entry_id": id, "file_name": "h.pdf", "mime_type": "application/pdf", - "content_base64": "JVBERi0xLjQgaHVnZQ=="}), + "content_base64": "JVBERi0xLjQgaHVnZQ==", + "sha256": ledger_store::journal::sha256_hex(b"%PDF-1.4 huge")}), ); pinned( &mut m, diff --git a/crates/ledger-mcp/tests/http_concurrency.rs b/crates/ledger-mcp/tests/http_concurrency.rs index a6499a3..6b607d6 100644 --- a/crates/ledger-mcp/tests/http_concurrency.rs +++ b/crates/ledger-mcp/tests/http_concurrency.rs @@ -119,7 +119,8 @@ async fn draft(c: &Client, tag: &str, cents: u64, with_receipt: bool) -> u64 { c, "books.attach_receipt", json!({"entry_id": id, "file_name": format!("{tag}.pdf"), - "mime_type": "application/pdf", "content_base64": b64(tag.as_bytes())}), + "mime_type": "application/pdf", "content_base64": b64(tag.as_bytes()), + "sha256": ledger_store::journal::sha256_hex(tag.as_bytes())}), ) .await .unwrap(); diff --git a/crates/ledger-mcp/tests/mcp_stdio.rs b/crates/ledger-mcp/tests/mcp_stdio.rs index 4343c16..54642be 100644 --- a/crates/ledger-mcp/tests/mcp_stdio.rs +++ b/crates/ledger-mcp/tests/mcp_stdio.rs @@ -126,8 +126,7 @@ impl Mcp { } } -/// A test's dir and what sits beside it (`.inbox`, `.elsewhere.txt`), -/// removed when the test ends, pass or panic. +/// A test's dir, removed when the test ends, pass or panic. struct TestDir(std::path::PathBuf); impl std::ops::Deref for TestDir { @@ -146,11 +145,6 @@ impl AsRef for TestDir { impl Drop for TestDir { fn drop(&mut self) { let _ = std::fs::remove_dir_all(&self.0); - for beside in [".inbox", ".elsewhere.txt"] { - let p = std::path::PathBuf::from(format!("{}{beside}", self.0.display())); - let _ = std::fs::remove_dir_all(&p); - let _ = std::fs::remove_file(&p); - } } } @@ -185,6 +179,9 @@ fn mcp_end_to_end_over_stdio() { let list = mcp.request("tools/list", json!({})); let tools = list["result"]["tools"].as_array().unwrap(); assert_eq!(tools.len(), 108, "expected 108 tools, got {}", tools.len()); + // Files come in as tool input: books.begin_upload, never a local path. + assert!(tools.iter().any(|t| t["name"] == "books.begin_upload")); + assert!(!tools.iter().any(|t| t["name"] == "books.import_local_file")); let chart = tools.iter().find(|t| t["name"] == "books.chart").unwrap(); assert_eq!(chart["annotations"]["readOnlyHint"], true); assert!( @@ -267,6 +264,7 @@ fn mcp_end_to_end_over_stdio() { "file_name": "rent.png", "mime_type": "image/png", "content_base64": png, + "sha256": sha(b"fake-png-bytes"), }), ); assert_eq!(attached["deduplicated"], false); @@ -461,6 +459,7 @@ fn mcp_assets_end_to_end_over_stdio() { "file_name": "bill-of-sale.pdf", "mime_type": "application/pdf", "content_base64": base64_like(b"fake bill of sale"), + "sha256": sha(b"fake bill of sale"), }), ); let aid = bill["attachment_id"].as_u64().unwrap(); @@ -628,7 +627,8 @@ fn mcp_receipt_corrections_over_stdio() { let att = mcp.call( "books.attach_receipt", json!({"entry_id": id, "file_name": "receipt.eml", "mime_type": "message/rfc822", - "content_base64": base64_like(receipt)}), + "content_base64": base64_like(receipt), + "sha256": sha(receipt)}), ); let posted = mcp.call( "books.post_entry", @@ -655,7 +655,8 @@ fn mcp_receipt_corrections_over_stdio() { let pdf = mcp.call( "books.attach_receipt", json!({"entry_id": big, "file_name": "receipt.pdf", "mime_type": "application/pdf", - "content_base64": base64_like(b"pdf-bytes")}), + "content_base64": base64_like(b"pdf-bytes"), + "sha256": sha(b"pdf-bytes")}), )["attachment_id"] .as_u64() .unwrap(); @@ -782,46 +783,6 @@ fn mcp_resources_over_stdio() { assert!(resp.get("error").is_some()); } -/// `--import-root` replaces the default import root (the downloads -/// folder): a file in the named folder is filed; one beside it is not. -#[test] -fn import_root_flag_replaces_the_defaults() { - let dir = test_dir("import-root"); - let inbox = dir.with_extension("inbox"); - std::fs::create_dir_all(&inbox).unwrap(); - let inside = inbox.join("receipt.txt"); - let temp = dir.with_extension("elsewhere.txt"); - for p in [&inside, &temp] { - std::fs::write(p, b"a receipt").unwrap(); - } - let mut mcp = Mcp::spawn_with( - &dir.join("books-data"), - &[std::ffi::OsStr::new("--import-root"), inbox.as_os_str()], - ); - mcp.request( - "initialize", - json!({ - "protocolVersion": "2025-06-18", - "capabilities": {}, - "clientInfo": {"name": "smoke-test", "version": "0.1.0"}, - }), - ); - mcp.notify("notifications/initialized"); - let ok = mcp.call( - "books.import_local_file", - json!({"path": inside.to_str().unwrap(), "category": "receipt"}), - ); - assert_eq!(ok["file_name"], "receipt.txt"); - let why = mcp.call_err( - "books.import_local_file", - json!({"path": temp.to_str().unwrap(), "category": "receipt"}), - ); - assert!( - why.contains("no such file in the server's import folders"), - "{why}" - ); -} - /// Start a session: initialize, then initialized. fn init(mcp: &mut Mcp) -> Value { let init = mcp.request( @@ -836,173 +797,25 @@ fn init(mcp: &mut Mcp) -> Value { init } -/// Through the real tools, inside a root (local_files.rs INVARIANTS 1-3, -/// Protected::of; S1 and S5 of the P0 review, I of round 2): the key -/// file in force, moved into the root after the server started (a root -/// holding the books or a key is refused at startup: startup.rs), is -/// sealed with what its link leads to and their folders, but never the -/// root itself; links in the root to the journal or a vault record lead -/// outside it, and get the one refusal a missing file gets (INVARIANT 2). -/// books.import_local_file and books.set_w9's w9_file_path agree. -#[test] -fn import_local_file_and_set_w9_refuse_the_key_vault_and_journal_inside_a_root() { - use std::os::unix::fs::symlink; - let dir = test_dir("sealed-in-root"); - let root = dir.join("root"); - let books = dir.join("books"); - let data = books.join("books-data"); - let tray = root.join("drop"); - let config = dir.join("config"); - for d in [ - &data, - &tray, - &root.join("alias"), - &root.join("hidden"), - &config.join("wanderbooks"), - ] { - std::fs::create_dir_all(d).unwrap(); - } - std::fs::write(root.join("hidden/real.key"), "ab".repeat(32)).unwrap(); - std::fs::write(root.join("hidden/notes.pdf"), b"%PDF beside the key").unwrap(); - std::fs::write(root.join("loose.key"), "ef".repeat(32)).unwrap(); - symlink(root.join("hidden/real.key"), root.join("alias/vault.key")).unwrap(); - std::fs::write(config.join("wanderbooks/vault.key"), "cd".repeat(32)).unwrap(); - let settings = |key: &std::path::Path| { - std::fs::write( - books.join("wanderbooks.toml"), - format!( - "[company]\nname = \"Sealed Co\"\n[vault]\nkey_file = \"{}\"\n", - key.display() - ), - ) - .unwrap(); - }; - // Started with the default key (outside the root), so it starts. - settings(&config.join("wanderbooks/vault.key")); - std::fs::write(tray.join("receipt.pdf"), b"%PDF a receipt").unwrap(); - std::fs::write(tray.join("w9.pdf"), b"%PDF\xff W-9 scan").unwrap(); - let root_arg = [std::ffi::OsStr::new("--import-root"), root.as_os_str()]; - let mut mcp = Mcp::spawn_env(&data, &root_arg, &[("XDG_CONFIG_HOME", &config)]); - init(&mut mcp); - mcp.call( - "books.add_party", - json!({"name": "Acme", "roles": ["contractor"]}), - ); - mcp.call( - "books.set_tin", - json!({"party": "Acme", "tin": "291-12-2846", "tin_type": "ssn"}), - ); - let journal = data.join(ledger_store::JOURNAL_FILE); - let vault_record = books - .join("vault") - .read_dir() - .unwrap() - .map(|e| e.unwrap().path()) - .find(|p| p.extension().is_some_and(|e| e == "bin")) - .unwrap(); - for (name, to) in [ - ("journal-link.pdf", journal.clone()), - ("record-link.pdf", vault_record.clone()), - ("default-key-link.pdf", config.join("wanderbooks/vault.key")), - ("dangling-link.pdf", config.join("wanderbooks/missing.key")), - ] { - symlink(&to, tray.join(name)).unwrap(); - } - // The key in force moves into the root (a link, alias/vault.key -> - // hidden/real.key); the settings reload. - settings(&root.join("alias/vault.key")); - let refuse = |mcp: &mut Mcp, path: &std::path::Path, why: &str| { - let path = path.to_str().unwrap(); - let err = mcp.call_err( - "books.import_local_file", - json!({"path": path, "category": "document"}), - ); - assert!(err.contains(why), "{path}: {err}"); - assert!(!err.contains(dir.to_str().unwrap()), "names a path: {err}"); - let err = mcp.call_err( - "books.set_w9", - json!({"party": "Acme", "received_on": "2026-01-03", "entity_type": "Individual", - "tin": "291-12-2846", "tin_type": "ssn", "w9_file_path": path}), - ); - assert!(err.contains(why), "{path}: {err}"); - }; - for p in [ - root.join("hidden/real.key"), - root.join("hidden/notes.pdf"), - root.join("alias/vault.key"), - ] { - refuse(&mut mcp, &p, "belongs to the books or the TIN vault"); - } - for p in [ - journal.clone(), - vault_record, - config.join("wanderbooks/vault.key"), - tray.join("journal-link.pdf"), - tray.join("record-link.pdf"), - tray.join("default-key-link.pdf"), - tray.join("dangling-link.pdf"), - tray.join("missing.pdf"), - ] { - refuse( - &mut mcp, - &p, - "no such file in the server's import folders (by default your downloads folder)", - ); - } - // A key kept loose in the root itself seals the key, not the root. - settings(&root.join("loose.key")); - refuse( - &mut mcp, - &root.join("loose.key"), - "belongs to the books or the TIN vault", - ); - // Ordinary files in the root are filed (with the key the vault was - // made with back in force). - mcp.call( - "books.import_local_file", - json!({"path": tray.join("receipt.pdf").to_str().unwrap(), "category": "receipt"}), - ); - // A link is filed under the name and type of the file it leads to. - std::fs::write(tray.join("photo.png"), b"\x89PNG a photo").unwrap(); - symlink(tray.join("photo.png"), tray.join("named-like.pdf")).unwrap(); - let filed = mcp.call( - "books.import_local_file", - json!({"path": tray.join("named-like.pdf").to_str().unwrap(), "category": "receipt"}), - ); - assert_eq!(filed["file_name"], "photo.png"); - let back = mcp.call( - "books.get_receipt", - json!({"attachment_id": filed["attachment_id"]}), - ); - assert_eq!(back["mime_type"], "image/png", "{back}"); - settings(&config.join("wanderbooks/vault.key")); - mcp.call( - "books.set_w9", - json!({"party": "Acme", "received_on": "2026-01-03", "entity_type": "Individual", - "tin": "291-12-2846", "tin_type": "ssn", - "w9_file_path": tray.join("w9.pdf").to_str().unwrap()}), - ); -} - /// Outputs that carry full TINs (books.get_w9, books.vendor_1099 with /// include_tins, books.export_cpa_package with TINs) are never written -/// into an import root, and a copy of one moved into a root is refused by -/// its bytes: a TIN-bearing file can't be filed back in as a receipt and +/// inside the books, and wherever they are written (a downloads folder +/// too), each is refused by its bytes when given back to be filed: a +/// TIN-bearing file can't be filed back in as a receipt or document and /// read out through books.get_receipt (S7 of the P0 review, 2026-10-16). #[test] fn tin_bearing_files_never_land_where_they_could_be_filed_back_in() { let dir = test_dir("tin-files"); - let root = dir.join("inbox"); - let out = dir.join("out"); + let books = dir.join("books"); + let downloads = dir.join("downloads"); let config = dir.join("config"); - for d in [&root, &out, &config] { + for d in [&downloads, &config] { std::fs::create_dir_all(d).unwrap(); } - std::fs::write(root.join("w9.pdf"), b"%PDF W-9 TIN 291-12-2846").unwrap(); - let root_arg = [std::ffi::OsStr::new("--import-root"), root.as_os_str()]; + let w9 = b"%PDF-1.4 W-9 TIN 291-12-2846"; let mut mcp = Mcp::spawn_env( - &dir.join("books").join("books-data"), - &root_arg, + &books.join("books-data"), + &[], &[("XDG_CONFIG_HOME", &config)], ); init(&mut mcp); @@ -1018,61 +831,92 @@ fn tin_bearing_files_never_land_where_they_could_be_filed_back_in() { "books.set_w9", json!({"party": "Acme", "received_on": "2026-01-03", "entity_type": "Individual", "tin": "291-12-2846", "tin_type": "ssn", - "w9_file_path": root.join("w9.pdf").to_str().unwrap()}), + "w9_content_base64": base64_like(w9), "w9_sha256": sha(w9)}), ); - let in_root = |name: &str| root.join(name).to_str().unwrap().to_string(); + // Inside the books: refused, and nothing written there. + let before = std::fs::read_dir(&books).unwrap().count(); + let in_books = |name: &str| books.join(name).to_str().unwrap().to_string(); for (tool, args) in [ ( "books.get_w9", - json!({"party": "Acme", "save_to": in_root("copy.pdf")}), + json!({"party": "Acme", "save_to": in_books("copy.pdf")}), ), ( "books.vendor_1099", - json!({"year": 2026, "include_tins": true, "save_csv_to": in_root("1099.csv")}), + json!({"year": 2026, "include_tins": true, "save_csv_to": in_books("1099.csv")}), ), ( "books.export_cpa_package", - json!({"cash_accounts": ["1000"], "save_to": in_root("cpa.zip")}), + json!({"cash_accounts": ["1000"], "save_to": in_books("cpa.zip")}), ), ( "books.export_cpa_package", - json!({"cash_accounts": ["1000"], "save_to": root.to_str().unwrap()}), + json!({"cash_accounts": ["1000"], "save_to": books.to_str().unwrap()}), ), ] { let why = mcp.call_err(tool, args); - assert!( - why.contains("never saved in the server's import folders"), - "{tool}: {why}" - ); + assert!(why.contains("inside the books"), "{tool}: {why}"); } assert_eq!( - std::fs::read_dir(&root).unwrap().count(), - 1, - "wrote into the root" + std::fs::read_dir(&books).unwrap().count(), + before, + "wrote into the books" ); - // Saved elsewhere, then moved into the root: refused by its bytes. - let w9 = out.join("copy.pdf"); + // Written to a downloads folder, each is refused by its bytes, by + // every way a file is filed. + let in_downloads = |name: &str| downloads.join(name).to_str().unwrap().to_string(); mcp.call( "books.get_w9", - json!({"party": "Acme", "save_to": w9.to_str().unwrap()}), + json!({"party": "Acme", "save_to": in_downloads("copy.pdf")}), ); - let csv = out.join("1099.csv"); mcp.call( "books.vendor_1099", - json!({"year": 2026, "include_tins": true, "save_csv_to": csv.to_str().unwrap()}), + json!({"year": 2026, "include_tins": true, "save_csv_to": in_downloads("1099.csv")}), ); - for (from, name) in [(&w9, "receipt.pdf"), (&csv, "statement.csv")] { - std::fs::copy(from, root.join(name)).unwrap(); - let why = mcp.call_err( - "books.import_local_file", - json!({"path": in_root(name), "category": "receipt"}), - ); - assert!(why.contains("wrote with full TINs"), "{why}"); + mcp.call( + "books.export_cpa_package", + json!({"cash_accounts": ["1000"], "save_to": in_downloads("cpa.zip")}), + ); + let written: Vec = std::fs::read_dir(&downloads) + .unwrap() + .map(|e| e.unwrap().path()) + .collect(); + assert_eq!(written.len(), 3, "{written:?}"); + let journal = std::fs::read(books.join("books-data").join(ledger_store::JOURNAL_FILE)).unwrap(); + for path in &written { + let bytes = std::fs::read(path).unwrap(); + for (tool, args) in [ + ( + "books.attach_receipt", + json!({"file_name": "receipt.pdf", "mime_type": "application/pdf", + "content_base64": base64_like(&bytes), + "sha256": sha(&bytes)}), + ), + ( + "books.file_document", + json!({"category": "statement", "file_name": "statement.csv", + "mime_type": "text/csv", + "content_base64": base64_like(&bytes), + "sha256": sha(&bytes)}), + ), + ] { + let why = mcp.call_err(tool, args); + assert!( + why.contains("wrote with full TINs"), + "{tool} {}: {why}", + path.display() + ); + } } + assert_eq!( + std::fs::read(books.join("books-data").join(ledger_store::JOURNAL_FILE)).unwrap(), + journal, + "a refusal stored something" + ); // Without TINs, the 1099 worksheet may go anywhere a file may. mcp.call( "books.vendor_1099", - json!({"year": 2026, "save_csv_to": in_root("plain.csv")}), + json!({"year": 2026, "save_csv_to": in_downloads("plain.csv")}), ); } @@ -1317,6 +1161,11 @@ fn handbook_tool_and_resources_over_stdio() { assert!(resp.get("error").is_some()); } +/// The SHA-256 (hex) a file given inline comes with. +fn sha(bytes: &[u8]) -> String { + ledger_store::journal::sha256_hex(bytes) +} + fn base64_like(bytes: &[u8]) -> String { const ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; let mut out = String::new(); @@ -1409,7 +1258,8 @@ fn mcp_statement_pdfs_and_cpa_package_over_stdio() { mcp.call( "books.attach_receipt", json!({"entry_id": id, "file_name": "r.pdf", "mime_type": "application/pdf", - "content_base64": base64_like(desc.as_bytes())}), + "content_base64": base64_like(desc.as_bytes()), + "sha256": sha(desc.as_bytes())}), ); mcp.call( "books.post_entry", @@ -1571,7 +1421,8 @@ fn mcp_period_statements_over_stdio() { mcp.call( "books.attach_receipt", json!({"entry_id": id, "file_name": "r.pdf", "mime_type": "application/pdf", - "content_base64": base64_like(b"r")}), + "content_base64": base64_like(b"r"), + "sha256": sha(b"r")}), ); mcp.call( "books.post_entry", @@ -2047,6 +1898,7 @@ fn mcp_payout_gross_up_over_stdio() { "file_name": "payout.pdf", "mime_type": "application/pdf", "content_base64": base64_like(b"%PDF-1.4 statement"), + "sha256": sha(b"%PDF-1.4 statement"), }), ); mcp.call( @@ -2133,6 +1985,7 @@ fn mcp_vendor_1099_over_stdio() { "file_name": "invoice.pdf", "mime_type": "application/pdf", "content_base64": base64_like(b"%PDF-1.4 invoice"), + "sha256": sha(b"%PDF-1.4 invoice"), }), ); mcp.call( diff --git a/crates/ledger-mcp/tests/oracle_mcp.rs b/crates/ledger-mcp/tests/oracle_mcp.rs index 7b0040e..6aad501 100644 --- a/crates/ledger-mcp/tests/oracle_mcp.rs +++ b/crates/ledger-mcp/tests/oracle_mcp.rs @@ -56,8 +56,7 @@ impl Drop for Mcp { static DIR_SEQ: AtomicU64 = AtomicU64::new(0); -/// A test's dir and the folder beside it (`.inbox`), -/// removed when the test ends, pass or panic. +/// A test's dir, removed when the test ends, pass or panic. struct TestDir(PathBuf); impl std::ops::Deref for TestDir { @@ -76,7 +75,6 @@ impl AsRef for TestDir { impl Drop for TestDir { fn drop(&mut self) { let _ = std::fs::remove_dir_all(&self.0); - let _ = std::fs::remove_dir_all(format!("{}.inbox", self.0.display())); } } @@ -109,14 +107,6 @@ fn books_toml(dir: &Path, threshold: &str) -> PathBuf { path } -/// The server's import root, beside the books (never the books dir): files -/// given to books.import_local_file. -fn inbox(dir: &Path) -> PathBuf { - let inbox = dir.with_extension("inbox"); - std::fs::create_dir_all(&inbox).unwrap(); - inbox -} - fn data_dir(dir: &Path) -> PathBuf { dir.join("books-data") } @@ -131,8 +121,6 @@ impl Mcp { .arg(data_dir(dir)) .arg("--books-toml") .arg(&toml) - .arg("--import-root") - .arg(inbox(dir)) .env("XDG_RUNTIME_DIR", &run) .stdin(Stdio::piped()) .stdout(Stdio::piped()) @@ -1737,7 +1725,7 @@ fn post_contribution(m: &mut Mcp) -> u64 { m.ok( "books.attach_receipt", json!({"entry_id": e, "file_name": "c.txt", "mime_type": "text/plain", - "content_base64": b64e(b"contribution")}), + "content_base64": b64e(b"contribution"), "sha256": sha256(b"contribution")}), ); m.ok( "books.post_entry", @@ -2118,11 +2106,24 @@ fn resources_read_back_as_the_matching_tools() { let want = m.ok(tool, args); assert_eq!(got, want, "{uri} reads as {tool}"); } + // A receipt reads as books.get_receipt's meta view: what it is, never + // its bytes (those come with view bytes). for (id, bytes) in [(r1, &b"first receipt"[..]), (r2, &b"second receipt"[..])] { let got = read(&mut m, &format!("books://receipts/{id}")); assert_eq!(got["attachment_id"], json!(id)); - assert_eq!(b64d(got["content_base64"].as_str().unwrap()), bytes); + assert!(got.get("content_base64").is_none(), "no bytes: {got}"); assert_eq!(got["sha256"], sha256(bytes)); + assert_eq!(got["size_bytes"], json!(bytes.len())); + assert_eq!( + got, + m.ok("books.get_receipt", json!({"attachment_id": id})), + "books://receipts/{id} reads as books.get_receipt" + ); + let full = m.ok( + "books.get_receipt", + json!({"attachment_id": id, "view": "bytes"}), + ); + assert_eq!(b64d(full["content_base64"].as_str().unwrap()), bytes); } let r = m.rpc("resources/read", json!({"uri": "books://nope"})); let err = r.get("error").unwrap_or_else(|| panic!("unknown URI: {r}")); @@ -2136,8 +2137,8 @@ fn resources_read_back_as_the_matching_tools() { /// Receipts and documents round-trip: the bytes come back exactly (SHA-256 /// of what was sent), with the name, MIME type, category, entry, and -/// description given. Kills: get_receipt's lookup, import_local_file's -/// document arm. +/// description given; only view bytes carries the bytes. Kills: +/// get_receipt's lookup and its views. #[test] fn receipts_and_documents_round_trip() { let dir = test_dir("files"); @@ -2150,48 +2151,27 @@ fn receipts_and_documents_round_trip() { let a = m.ok( "books.attach_receipt", json!({"entry_id": e, "file_name": "first.pdf", "mime_type": "application/pdf", - "content_base64": b64e(first), "description": "Receipt ID 1007; Paid from: Bluevine"}), + "content_base64": b64e(first), "sha256": sha256(first), + "description": "Receipt ID 1007; Paid from: Bluevine"}), ); let b = m.ok( "books.file_document", json!({"category": "legal", "file_name": "oa.png", "mime_type": "image/png", - "content_base64": b64e(second), "description": "Operating agreement"}), + "content_base64": b64e(second), "sha256": sha256(second), + "description": "Operating agreement"}), ); - // Local files come from outside the books, in the server's import - // folder (local_files.rs). - let inbox = inbox(&dir); - let local_doc = inbox.join("ein-letter.pdf"); - std::fs::write(&local_doc, b"EIN letter bytes").unwrap(); let c = m.ok( - "books.import_local_file", - json!({"path": local_doc.to_str().unwrap(), "category": "document", - "document_category": "tax", "description": "IRS CP 575"}), + "books.file_document", + json!({"category": "tax", "file_name": "ein-letter.pdf", "mime_type": "application/pdf", + "content_base64": b64e(b"EIN letter bytes"), "sha256": sha256(b"EIN letter bytes"), + "description": "IRS CP 575"}), ); - assert_eq!(c["file_name"], "ein-letter.pdf"); - let local_receipt = inbox.join("scan.png"); - std::fs::write(&local_receipt, b"a scanned receipt").unwrap(); let r = m.ok( - "books.import_local_file", - json!({"path": local_receipt.to_str().unwrap(), "category": "receipt", - "entry_id": e, "file_name": "renamed.png", "description": "scan"}), + "books.attach_receipt", + json!({"entry_id": e, "file_name": "renamed.png", "mime_type": "image/png", + "content_base64": b64e(b"a scanned receipt"), + "sha256": sha256(b"a scanned receipt"), "description": "scan"}), ); - assert_eq!(r["file_name"], "renamed.png"); - m.refuse( - "books.import_local_file", - json!({"path": local_receipt.to_str().unwrap(), "category": "photo"}), - ); - // Nothing of the books is ever read back in through a path: outside - // the import folders, it gets the one refusal any path there gets. - let journal = data_dir(&dir).join(ledger_store::JOURNAL_FILE); - for p in [journal, dir.join("wanderbooks-1000000.00.toml")] { - assert!(p.exists()); - let why = m.refuse( - "books.import_local_file", - json!({"path": p.to_str().unwrap(), "category": "document"}), - ); - assert!(why.contains("import folders"), "{why}"); - assert!(!why.contains(dir.to_str().unwrap()), "names a path: {why}"); - } // (result, bytes, file name, MIME type, entry, description) type Case<'a> = (&'a Value, &'a [u8], &'a str, &'a str, Option, &'a str); @@ -2233,13 +2213,24 @@ fn receipts_and_documents_round_trip() { for (out, bytes, name, mime, entry, desc) in cases { let id = out["attachment_id"].as_u64().unwrap(); assert_eq!(out["sha256"], sha256(bytes), "{name}: sha256 of the bytes"); - let got = m.ok("books.get_receipt", json!({"attachment_id": id})); + let meta = m.ok("books.get_receipt", json!({"attachment_id": id})); + assert!( + meta.get("content_base64").is_none(), + "{name}: meta, no bytes" + ); + let got = m.ok( + "books.get_receipt", + json!({"attachment_id": id, "view": "bytes"}), + ); assert_eq!(got["attachment_id"], json!(id)); assert_eq!( b64d(got["content_base64"].as_str().unwrap()), bytes, "{name}" ); + let mut without = got.clone(); + without.as_object_mut().unwrap().remove("content_base64"); + assert_eq!(without, meta, "{name}: view bytes is meta and the bytes"); assert_eq!(got["sha256"], sha256(bytes)); assert_eq!(got["file_name"], name); assert_eq!(got["mime_type"], mime); @@ -2263,19 +2254,26 @@ fn receipts_and_documents_round_trip() { assert_eq!(cat(&b), "legal"); assert_eq!(cat(&c), "tax"); m.refuse("books.get_receipt", json!({"attachment_id": 999})); + for view in ["", "BYTES", "content", "meta "] { + let why = m.refuse( + "books.get_receipt", + json!({"attachment_id": a["attachment_id"], "view": view}), + ); + assert!( + why.contains("meta (the default) or bytes"), + "{view:?}: {why}" + ); + } } -/// 25 MB (MiB) is the most a receipt or document may be: exactly the -/// limit is accepted through every way in, one byte more refused (checked -/// on the local-file path, where it costs no upload). Kills: the limit -/// comparisons in attach_receipt, file_document, and import_local_file, -/// and import_local_file's limit arithmetic (2 MiB must pass). +/// 256 KiB is the most a file may be given inline (plan 2.1): exactly +/// the limit is accepted by books.attach_receipt and books.file_document +/// and stored whole, one byte more is refused by both, pointing at +/// books.begin_upload, and nothing is stored. Kills: the limit comparison +/// in take_file. #[test] -fn uploads_up_to_25_mib_are_accepted_and_no_more() { - const LIMIT: usize = 25 * 1024 * 1024; - // Over 50 MB of books: removed however the test ends, with its inbox - // (the system temp dir is often RAM, and a mutation run starts this - // test hundreds of times). +fn inline_files_up_to_256_kib_are_accepted_and_no_more() { + const LIMIT: usize = 256 * 1024; let dir = test_dir("limits"); let s = new_store(&dir); drop(s); @@ -2284,18 +2282,19 @@ fn uploads_up_to_25_mib_are_accepted_and_no_more() { let out = m.ok( "books.attach_receipt", json!({"file_name": "big.bin", "mime_type": "application/octet-stream", - "content_base64": b64e(&at)}), + "content_base64": b64e(&at), "sha256": sha256(&at)}), ); let at_id = out["attachment_id"].as_u64().unwrap(); let doc = vec![b'b'; LIMIT]; let out = m.ok( "books.file_document", json!({"category": "other", "file_name": "big.bin", - "mime_type": "application/octet-stream", "content_base64": b64e(&doc)}), + "mime_type": "application/octet-stream", "content_base64": b64e(&doc), + "sha256": sha256(&doc)}), ); let doc_id = out["attachment_id"].as_u64().unwrap(); let listed = m.ok("books.list_receipts", json!({})); - for id in [at_id, doc_id] { + for (id, bytes) in [(at_id, &at), (doc_id, &doc)] { let row = listed["receipts"] .as_array() .unwrap() @@ -2304,19 +2303,195 @@ fn uploads_up_to_25_mib_are_accepted_and_no_more() { .unwrap() .clone(); assert_eq!(row["size_bytes"], json!(LIMIT), "stored whole"); + // And it comes back inline, whole. + let got = m.ok( + "books.get_receipt", + json!({"attachment_id": id, "view": "bytes"}), + ); + assert_eq!(&b64d(got["content_base64"].as_str().unwrap()), bytes); } + let journal = std::fs::read(data_dir(&dir).join(ledger_store::JOURNAL_FILE)).unwrap(); + let over = vec![b'c'; LIMIT + 1]; + for (tool, args) in [ + ( + "books.attach_receipt", + json!({"file_name": "over.bin", "mime_type": "application/octet-stream", + "content_base64": b64e(&over), "sha256": sha256(&over)}), + ), + ( + "books.file_document", + json!({"category": "other", "file_name": "over.bin", + "mime_type": "application/octet-stream", "content_base64": b64e(&over), + "sha256": sha256(&over)}), + ), + ] { + let why = m.refuse(tool, args); + assert!( + why.contains("books.begin_upload") && why.contains(&(LIMIT + 1).to_string()), + "{tool}: {why}" + ); + } + assert_eq!( + std::fs::read(data_dir(&dir).join(ledger_store::JOURNAL_FILE)).unwrap(), + journal, + "a refusal stored something" + ); +} - for (n, accepted) in [(2 * 1024 * 1024, true), (LIMIT, true), (LIMIT + 1, false)] { - let path = inbox(&dir).join(format!("local-{n}.bin")); - std::fs::write(&path, vec![b'c' ^ (n % 7) as u8; n]).unwrap(); - let args = json!({"path": path.to_str().unwrap(), "category": "receipt"}); - if accepted { - m.ok("books.import_local_file", args); - } else { - m.refuse("books.import_local_file", args); +/// The bytes a filing tool takes are inline base64 with its own SHA-256, +/// or an upload; never both, never neither. A sha256 missing, or not the +/// bytes' own (a byte lost on the way), or base64 that isn't, is refused +/// (guard base64_corrupt); inline bytes need their file_name and +/// mime_type. Over stdio there are no uploads: an upload_id is refused, +/// and books.begin_upload answers only for bytes stored already (the +/// attachment holding them). Every refusal stores nothing. +#[test] +fn inline_bytes_come_with_their_own_sha256_and_uploads_need_the_server() { + let dir = test_dir("inline"); + let s = new_store(&dir); + drop(s); + let mut m = Mcp::start(&dir, "1000000.00"); + let bytes: &[u8] = b"%PDF-1.4 a receipt"; + let other: &[u8] = b"%PDF-1.4 a receipt!"; + let journal = || std::fs::read(data_dir(&dir).join(ledger_store::JOURNAL_FILE)).unwrap(); + let before = journal(); + for (tool, base) in [ + ("books.attach_receipt", json!({})), + ("books.file_document", json!({"category": "other"})), + ] { + let with = |extra: Value| { + let mut args = base.clone(); + for (k, v) in json!({"file_name": "r.pdf", "mime_type": "application/pdf"}) + .as_object() + .unwrap() + .iter() + .chain(extra.as_object().unwrap()) + { + args[k] = v.clone(); + } + args + }; + for (args, said) in [ + // Neither, both. + (with(json!({})), "books.begin_upload"), + ( + with(json!({"upload_id": "u1", "content_base64": b64e(bytes), + "sha256": sha256(bytes)})), + "not both", + ), + // The sha256 missing, another file's, not hex; base64 that isn't. + (with(json!({"content_base64": b64e(bytes)})), "sha256"), + ( + with(json!({"content_base64": b64e(bytes), "sha256": sha256(other)})), + "base64_corrupt", + ), + ( + with(json!({"content_base64": b64e(bytes), "sha256": "not hex"})), + "base64_corrupt", + ), + ( + with(json!({"content_base64": "!!not base64", "sha256": sha256(bytes)})), + "base64_corrupt", + ), + // An upload, over stdio. + (with(json!({"upload_id": "u1"})), "takes files inline only"), + ] { + let why = m.refuse(tool, args.clone()); + assert!(why.contains(said), "{tool} {args}: {why}"); + } + // Inline bytes need their name and type. + for missing in ["file_name", "mime_type"] { + let mut args = with(json!({"content_base64": b64e(bytes), "sha256": sha256(bytes)})); + args.as_object_mut().unwrap().remove(missing); + let why = m.refuse(tool, args); + assert!(why.contains(missing), "{tool} without {missing}: {why}"); } - std::fs::remove_file(&path).unwrap(); } + // A sha256 given in capitals, or with spaces around it, is the same + // hash. + let out = m.ok( + "books.attach_receipt", + json!({"file_name": "r.pdf", "mime_type": "application/pdf", + "content_base64": b64e(bytes), + "sha256": format!(" {} ", sha256(bytes).to_ascii_uppercase())}), + ); + assert_eq!(out["sha256"], sha256(bytes)); + let id = out["attachment_id"].as_u64().unwrap(); + assert_ne!(journal(), before); + // begin_upload: the bytes stored already name their attachment; new + // ones need the books server. + let up = |sha: String| { + json!({"sha256": sha, "size": bytes.len(), "mime_type": "application/pdf", + "file_name": "r.pdf", "purpose": "receipt"}) + }; + let got = m.ok("books.begin_upload", up(sha256(bytes))); + assert_eq!(got["existing_attachment_id"], json!(id), "{got}"); + assert!(got.get("upload_id").is_none(), "{got}"); + assert!(got.get("command").is_none(), "{got}"); + let got = m.ok("books.begin_upload", up(sha256(bytes).to_ascii_uppercase())); + assert_eq!(got["existing_attachment_id"], json!(id), "{got}"); + let stored = journal(); + let why = m.refuse("books.begin_upload", up(sha256(other))); + assert!(why.contains("takes files inline only"), "{why}"); + // The CSV imports take csv_text or an upload, never both, never + // neither; an upload over stdio is refused. + for (tool, base) in [ + ( + "books.import_bank_csv", + json!({"bank_account_code": "1000", "default_offset_code": "4000"}), + ), + ("books.import_chart_csv", json!({})), + ] { + for (extra, said) in [ + (json!({}), "csv_text"), + (json!({"csv_text": "a,b\n", "upload_id": "u1"}), "not both"), + (json!({"upload_id": "u1"}), "takes files inline only"), + ] { + let mut args = base.clone(); + for (k, v) in extra.as_object().unwrap() { + args[k] = v.clone(); + } + let why = m.refuse(tool, args.clone()); + assert!(why.contains(said), "{tool} {args}: {why}"); + } + } + assert_eq!(journal(), stored, "a refusal stored something"); +} + +/// books.get_receipt's view bytes gives the bytes of a file of at most +/// 256 KiB; a larger one (filed before, or by an upload) is refused over +/// stdio, where there is no download, while its meta view still answers. +#[test] +fn get_receipt_gives_bytes_up_to_256_kib_over_stdio() { + const LIMIT: usize = 256 * 1024; + let dir = test_dir("get-bytes"); + let mut s = new_store(&dir); + let at = vec![b'x'; LIMIT]; + let over = vec![b'y'; LIMIT + 1]; + let (at_id, _, _) = s + .attach_receipt(None, "at.bin", "application/octet-stream", &at, None) + .unwrap(); + let (over_id, _, _) = s + .attach_receipt(None, "over.bin", "application/octet-stream", &over, None) + .unwrap(); + drop(s); + let mut m = Mcp::start(&dir, "1000000.00"); + let got = m.ok( + "books.get_receipt", + json!({"attachment_id": at_id, "view": "bytes"}), + ); + assert_eq!(b64d(got["content_base64"].as_str().unwrap()), at); + assert!(got.get("download").is_none(), "{got}"); + let why = m.refuse( + "books.get_receipt", + json!({"attachment_id": over_id, "view": "bytes"}), + ); + assert!(why.contains(&(LIMIT + 1).to_string()), "{why}"); + let meta = m.ok("books.get_receipt", json!({"attachment_id": over_id})); + assert_eq!(meta["size_bytes"], json!(LIMIT + 1)); + assert_eq!(meta["sha256"], sha256(&over)); + assert!(meta.get("content_base64").is_none(), "{meta}"); + assert!(meta.get("download").is_none(), "{meta}"); } /// Bank CSV import: money in debits the bank, money out credits it; the diff --git a/crates/ledger-mcp/tests/spec_assets.rs b/crates/ledger-mcp/tests/spec_assets.rs index 916e857..b73f17e 100644 --- a/crates/ledger-mcp/tests/spec_assets.rs +++ b/crates/ledger-mcp/tests/spec_assets.rs @@ -360,7 +360,9 @@ impl Mcp { self.ok( "books.file_document", json!({"category": "legal", "file_name": "bill-of-sale.txt", "mime_type": "text/plain", - "content_base64": b64(b"bill of sale"), "description": "Bill of sale"}), + "content_base64": b64(b"bill of sale"), + "sha256": ledger_store::journal::sha256_hex(b"bill of sale"), + "description": "Bill of sale"}), )["attachment_id"] .as_u64() .unwrap() diff --git a/crates/ledger-mcp/tests/spec_attachments.rs b/crates/ledger-mcp/tests/spec_attachments.rs index b3c7ce0..3331ce3 100644 --- a/crates/ledger-mcp/tests/spec_attachments.rs +++ b/crates/ledger-mcp/tests/spec_attachments.rs @@ -371,6 +371,11 @@ fn b64(bytes: &[u8]) -> String { base64::engine::general_purpose::STANDARD.encode(bytes) } +/// The SHA-256 (hex) a file given inline comes with. +fn sha(bytes: &[u8]) -> String { + ledger_store::journal::sha256_hex(bytes) +} + const CHART: &[(&str, &str, &str)] = &[ ("1000", "Bank", "asset"), ("1300", "Prepaid Expenses", "asset"), @@ -406,7 +411,7 @@ impl Mcp { self.ok( "books.file_document", json!({"category": category, "file_name": format!("{category}.pdf"), - "mime_type": "application/pdf", "content_base64": b64(bytes), + "mime_type": "application/pdf", "content_base64": b64(bytes), "sha256": sha(bytes), "link_to": link_to}), )["attachment_id"] .as_u64() @@ -416,7 +421,7 @@ impl Mcp { /// Attach a receipt (to `entry`, or unlinked); its id. fn receipt(&mut self, entry: Option, bytes: &[u8]) -> u64 { let mut args = json!({"file_name": "r.pdf", "mime_type": "application/pdf", - "content_base64": b64(bytes)}); + "content_base64": b64(bytes), "sha256": sha(bytes)}); if let Some(e) = entry { args["entry_id"] = json!(e); } @@ -787,25 +792,25 @@ fn filing_links_in_the_same_call_all_or_nothing() { m.refuse( "books.attach_receipt", json!({"entry_id": e, "file_name": "r.pdf", "mime_type": "application/pdf", - "content_base64": b64(b"r"), "link_to": link_to}), + "content_base64": b64(b"r"), "sha256": sha(b"r"), "link_to": link_to}), ); } m.refuse( "books.file_document", json!({"category": "contract", "file_name": "c.pdf", "mime_type": "application/pdf", - "content_base64": b64(b"c"), "link_to": [{"kind": "asset", "id": "NOPE"}]}), + "content_base64": b64(b"c"), "sha256": sha(b"c"), "link_to": [{"kind": "asset", "id": "NOPE"}]}), ); // An unknown entry_id is refused before anything is filed. m.refuse( "books.attach_receipt", json!({"entry_id": 999, "file_name": "r.pdf", "mime_type": "application/pdf", - "content_base64": b64(b"r"), "link_to": [{"kind": "company"}]}), + "content_base64": b64(b"r"), "sha256": sha(b"r"), "link_to": [{"kind": "company"}]}), ); assert_eq!(count(&mut m), n, "nothing filed"); let out = m.ok( "books.attach_receipt", json!({"entry_id": e, "file_name": "r.pdf", "mime_type": "application/pdf", - "content_base64": b64(b"r"), + "content_base64": b64(b"r"), "sha256": sha(b"r"), "link_to": [{"kind": "entry", "id": other}, {"kind": "party", "id": "acme design"}]}), ); assert_eq!( @@ -819,7 +824,7 @@ fn filing_links_in_the_same_call_all_or_nothing() { let doc = m.ok( "books.file_document", json!({"category": "formation", "file_name": "cert.pdf", "mime_type": "application/pdf", - "content_base64": b64(b"certificate"), "link_to": [{"kind": "company"}]}), + "content_base64": b64(b"certificate"), "sha256": sha(b"certificate"), "link_to": [{"kind": "company"}]}), ); assert_eq!(kinds(&doc), ["company"]); assert_eq!(m.count("document.filed"), 1); @@ -849,7 +854,7 @@ fn documents_file_under_the_attachment_kinds() { m.refuse( "books.file_document", json!({"category": "receipt-ish", "file_name": "x", "mime_type": "text/plain", - "content_base64": b64(b"x")}), + "content_base64": b64(b"x"), "sha256": sha(b"x")}), ); } diff --git a/crates/ledger-mcp/tests/spec_mcp.rs b/crates/ledger-mcp/tests/spec_mcp.rs index 31a223f..87d82f9 100644 --- a/crates/ledger-mcp/tests/spec_mcp.rs +++ b/crates/ledger-mcp/tests/spec_mcp.rs @@ -213,14 +213,6 @@ fn test_dir(name: &str) -> TestDir { /// A directory beside the books dir `dir`, outside it (vault keys, files /// with TINs). -/// The import root beside the books dir `dir`: files the tests hand the -/// server by path (books.import_local_file, books.set_w9 w9_file_path). -fn inbox(dir: &Path) -> PathBuf { - let d = dir.with_extension("inbox"); - std::fs::create_dir_all(&d).unwrap(); - d -} - fn outside(dir: &Path) -> PathBuf { let d = PathBuf::from(format!("{}-outside", dir.display())); std::fs::create_dir_all(&d).unwrap(); @@ -260,10 +252,6 @@ fn spawn_child(dir: &Path, toml: &Path) -> Child { .into_iter() .flatten(), ) - // Local files come only from here (the default is the downloads - // folder): never the books, never where the server writes. - .arg("--import-root") - .arg(inbox(dir)) // books.show writes a viewer request here; keep it out of the // user's real runtime dir. .env("XDG_RUNTIME_DIR", &run) @@ -548,6 +536,7 @@ impl Mcp { "file_name": "receipt.txt", "mime_type": "text/plain", "content_base64": b64(bytes), + "sha256": sha(bytes), }); if let Some(e) = entry { args["entry_id"] = json!(e); @@ -660,6 +649,11 @@ fn canon(s: &str) -> String { } } +/// The SHA-256 (hex) a file given inline comes with. +fn sha(bytes: &[u8]) -> String { + ledger_store::journal::sha256_hex(bytes) +} + fn b64(bytes: &[u8]) -> String { const T: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; let mut out = String::new(); @@ -811,12 +805,12 @@ fn happy_path(name: &str) -> Mcp { json!({"party": "Landlord", "received_on": "2026-01-03", "entity_type": "LLC (partnership)", "tin_last4": "4321", "kept_at": "CPA portal"}), ); - let w9_pdf = inbox(&dir).join("landlord-w9.pdf"); - std::fs::write(&w9_pdf, b"%PDF-1.4 W-9 EIN 79-9714321").unwrap(); + let w9_pdf = b"%PDF-1.4 W-9 EIN 79-9714321"; m.ok( "books.set_w9", json!({"party": "Landlord", "received_on": "2026-01-03", "entity_type": "LLC (partnership)", - "tin": "79-9714321", "tin_type": "ein", "w9_file_path": w9_pdf.to_str().unwrap()}), + "tin": "79-9714321", "tin_type": "ein", + "w9_content_base64": b64(w9_pdf), "w9_sha256": sha(w9_pdf)}), ); m.ok( "books.get_w9", @@ -889,17 +883,23 @@ fn happy_path(name: &str) -> Mcp { m.ok("books.list_receipts", json!({"include_withdrawn": true})); m.ok("books.list_receipts", json!({"entry_id": e2})); m.ok("books.get_receipt", json!({"attachment_id": r_old})); + m.ok( + "books.get_receipt", + json!({"attachment_id": r_old, "view": "bytes"}), + ); let doc = m.ok( "books.file_document", json!({"category": "legal", "file_name": "agreement.txt", "mime_type": "text/plain", - "content_base64": b64(b"operating agreement"), "description": "OA"}), + "content_base64": b64(b"operating agreement"), + "sha256": sha(b"operating agreement"), "description": "OA"}), )["attachment_id"] .as_u64() .unwrap(); let filed = m.ok( "books.file_document", json!({"category": "contract", "file_name": "lease.txt", "mime_type": "text/plain", - "content_base64": b64(b"lease"), "link_to": [{"kind": "company"}]}), + "content_base64": b64(b"lease"), + "sha256": sha(b"lease"), "link_to": [{"kind": "company"}]}), )["attachment_id"] .as_u64() .unwrap(); @@ -915,13 +915,15 @@ fn happy_path(name: &str) -> Mcp { "books.unlink_attachment", json!({"attachment_id": filed, "subject": {"kind": "company"}, "reason": "a lease, not a formation document"}), ); - // From outside the books, in the server's import folder. - let local = inbox(&dir).join("local-receipt.txt"); - std::fs::write(&local, b"a local receipt").unwrap(); - m.ok( - "books.import_local_file", - json!({"path": local.to_str().unwrap(), "category": "receipt"}), + // An upload of bytes already stored: begin_upload names the + // attachment that holds them (over stdio, the only upload answered). + let local = m.attach(None, b"a local receipt"); + let up = m.ok( + "books.begin_upload", + json!({"sha256": sha(b"a local receipt"), "size": 15, "mime_type": "text/plain", + "file_name": "local-receipt.txt", "purpose": "receipt"}), ); + assert_eq!(up["existing_attachment_id"], json!(local), "{up}"); // Reversal: preview, then confirmed. let e3 = m.draft( @@ -1071,7 +1073,8 @@ fn happy_path(name: &str) -> Mcp { let source = m.ok( "books.file_document", json!({"category": "import_source", "file_name": "bank.csv", "mime_type": "text/csv", - "content_base64": b64(csv.as_bytes())}), + "content_base64": b64(csv.as_bytes()), + "sha256": sha(csv.as_bytes())}), )["attachment_id"] .as_u64() .unwrap(); @@ -1092,7 +1095,8 @@ fn happy_path(name: &str) -> Mcp { let stmt = m.ok( "books.file_document", json!({"category": "statement", "file_name": "2026-01.pdf", "mime_type": "application/pdf", - "content_base64": b64(b"%PDF statement")}), + "content_base64": b64(b"%PDF statement"), + "sha256": sha(b"%PDF statement")}), )["attachment_id"] .as_u64() .unwrap(); @@ -1572,7 +1576,8 @@ fn run_op(m: &mut Mcp, k: &mut Known, op: &Op) { let out = m.tool( "books.attach_receipt", json!({"entry_id": e, "file_name": "r.txt", "mime_type": "text/plain", - "content_base64": b64(bytes.as_bytes())}), + "content_base64": b64(bytes.as_bytes()), + "sha256": sha(bytes.as_bytes())}), ); k.attachments.extend(new_id(&out, "attachment_id")); } @@ -1601,11 +1606,21 @@ fn run_op(m: &mut Mcp, k: &mut Known, op: &Op) { json!({"entry_id": entry.resolve(&k.entries), "date": date, "description": "undo", "confirmed": opt(*confirmed)}), ), - Op::Attach { entry, content } => ( - "books.attach_receipt", - json!({"entry_id": entry.map(|p| p.resolve(&k.entries)), "file_name": "r.bin", - "mime_type": "application/octet-stream", "content_base64": content}), - ), + Op::Attach { entry, content } => { + // The bytes' own SHA-256 (content that is no base64 has none). + let sha256 = { + use base64::Engine as _; + base64::engine::general_purpose::STANDARD + .decode(content) + .map_or_else(|_| "0".repeat(64), |b| sha(&b)) + }; + ( + "books.attach_receipt", + json!({"entry_id": entry.map(|p| p.resolve(&k.entries)), "file_name": "r.bin", + "mime_type": "application/octet-stream", "content_base64": content, + "sha256": sha256}), + ) + } Op::Link { att, entry } => ( "books.link_receipt", json!({"attachment_id": att.resolve(&k.attachments), diff --git a/crates/ledger-mcp/tests/spec_vault.rs b/crates/ledger-mcp/tests/spec_vault.rs index a480d2f..a6f2840 100644 --- a/crates/ledger-mcp/tests/spec_vault.rs +++ b/crates/ledger-mcp/tests/spec_vault.rs @@ -72,7 +72,7 @@ impl Drop for Mcp { static DIR_SEQ: AtomicU64 = AtomicU64::new(0); -/// A test's dir and the folders beside it (`-inbox`, `-outside`), +/// A test's dir and the folder beside it (`-outside`), /// removed when the test ends, pass or panic. struct TestDir(PathBuf); @@ -92,11 +92,7 @@ impl AsRef for TestDir { impl Drop for TestDir { fn drop(&mut self) { let _ = std::fs::remove_dir_all(&self.0); - for beside in ["-inbox", "-outside"] { - let p = PathBuf::from(format!("{}{beside}", self.0.display())); - let _ = std::fs::remove_dir_all(&p); - let _ = std::fs::remove_file(&p); - } + let _ = std::fs::remove_dir_all(format!("{}-outside", self.0.display())); } } @@ -119,12 +115,15 @@ fn outside(dir: &Path) -> PathBuf { d } -/// The server's import root, beside the books dir `dir`: W-9 files handed -/// to books.set_w9 by path. Never where the server writes TINs. -fn inbox(dir: &Path) -> PathBuf { - let d = PathBuf::from(format!("{}-inbox", dir.display())); - std::fs::create_dir_all(&d).unwrap(); - d +/// `bytes` as a tool takes them inline: base64. +fn b64(bytes: &[u8]) -> String { + use base64::Engine as _; + base64::engine::general_purpose::STANDARD.encode(bytes) +} + +/// The SHA-256 (hex) a file given inline comes with. +fn sha(bytes: &[u8]) -> String { + ledger_store::journal::sha256_hex(bytes) } /// `wanderbooks.toml` with a company name, the receipt threshold, and @@ -169,9 +168,6 @@ impl Mcp { .into_iter() .flatten() }) - // Files handed over by path come only from here. - .arg("--import-root") - .arg(inbox(dir)) .env("XDG_RUNTIME_DIR", &run) // Never the user's real vault key (the default key file is // under XDG_CONFIG_HOME); the tests name theirs anyway. @@ -519,11 +515,9 @@ impl Mcp { } self.tool("books.set_w9", args) } - /// A W-9 file outside the books; it shows the TIN, like the real one. - fn w9_file(&self, name: &str, tin: &str) -> PathBuf { - let p = inbox(&self.books_dir()).join(name); - std::fs::write(&p, format!("%PDF-1.4 Form W-9 TIN {tin}")).unwrap(); - p + /// A W-9 file (a PDF); it shows the TIN, like the real one. + fn w9_file(&self, tin: &str) -> Vec { + format!("%PDF-1.4 Form W-9 TIN {tin}").into_bytes() } /// Pay `vendor` `amount` from the bank in 2026, tagged. fn pay(&mut self, vendor: &str, amount: &str) { @@ -626,9 +620,9 @@ fn set_w9_refusals_write_nothing_anywhere() { let mut m = books("refusals"); let acme = m.party_id("Acme Design"); m.add_party("IRS", &["tax_agency"]); - let missing = inbox(&m.books_dir()).join("no-such-w9.pdf"); - let txt = inbox(&m.books_dir()).join("w9.txt"); - std::fs::write(&txt, "W-9").unwrap(); + let pdf = m.w9_file(SSN); + let txt = b"W-9"; + let big = [b"%PDF-1.4 ".as_slice(), &vec![b' '; 256 * 1024]].concat(); for (party, extra) in [ ("Acme Design", json!({"tin": "57193284", "tin_type": "ssn"})), // eight digits ( @@ -653,14 +647,6 @@ fn set_w9_refusals_write_nothing_anywhere() { "Acme Design", json!({"tin": 291122846.0, "tin_type": "ssn"}), ), // a float - ( - "Acme Design", - json!({"tin": SSN, "tin_type": "ssn", "w9_file_path": missing.to_str().unwrap()}), - ), - ( - "Acme Design", - json!({"tin": SSN, "tin_type": "ssn", "w9_file_path": txt.to_str().unwrap()}), - ), ( "Acme Design", json!({"tin": SSN, "tin_type": "ssn", "clear": true}), @@ -680,6 +666,51 @@ fn set_w9_refusals_write_nothing_anywhere() { out.err_text() ); } + // The W-9 file, each refused for its own reason. + let long = "!".repeat((256 * 1024usize).div_ceil(3) * 4 + 4); + for (extra, why) in [ + ( + json!({"w9_content_base64": b64(txt), "w9_sha256": sha(txt)}), + "a PDF, PNG, or JPEG", + ), + ( + json!({"w9_content_base64": b64(&pdf)}), + "comes with its sha256", + ), + ( + json!({"w9_content_base64": b64(&pdf), "w9_sha256": sha(txt)}), + "lost on the way", + ), + ( + json!({"w9_content_base64": "!!not base64", "w9_sha256": sha(&pdf)}), + "not valid base64", + ), + ( + json!({"w9_content_base64": b64(&big), "w9_sha256": sha(&big)}), + "upload it with books.begin_upload", + ), + ( + json!({"w9_content_base64": long, "w9_sha256": sha(&big)}), + "inline is at most", + ), + (json!({"w9_upload_id": "u1"}), "takes files inline only"), + ( + json!({"w9_upload_id": "u1", "w9_content_base64": b64(&pdf), "w9_sha256": sha(&pdf)}), + "not both", + ), + ( + json!({"w9_content_base64": b64(&pdf), "w9_sha256": sha(&pdf), "w9_file_name": "../w9.pdf"}), + "no folder", + ), + ] { + let mut args = json!({"tin": SSN, "tin_type": "ssn"}); + for (k, v) in extra.as_object().unwrap() { + args[k] = v.clone(); + } + let why_given = m.w9("Acme Design", args).err_text(); + assert!(why_given.contains(why), "{extra}: {why_given}"); + assert!(!holds_tin(why_given.as_bytes(), SSN_DIGITS), "{why_given}"); + } assert!( !m.vault().join(format!("tin-{acme}.bin")).exists(), "no record written" @@ -739,10 +770,10 @@ fn the_facts_never_disagree_with_the_vault() { fn the_w9_file_goes_to_the_vault_never_the_documents() { let mut m = books("w9-file"); let acme = m.party_id("Acme Design"); - let src = m.w9_file("acme-w9.pdf", SSN); + let src = m.w9_file(SSN); let Out::Ok(p) = m.w9( "Acme Design", - json!({"tin": SSN, "tin_type": "ssn", "w9_file_path": src.to_str().unwrap()}), + json!({"tin": SSN, "tin_type": "ssn", "w9_content_base64": b64(&src), "w9_sha256": sha(&src)}), ) else { panic!("set_w9 with a file") }; @@ -760,8 +791,9 @@ fn the_w9_file_goes_to_the_vault_never_the_documents() { json!({"party": "Acme Design", "save_to": to.to_str().unwrap()}), ); assert_eq!(out["path"], json!(to.display().to_string())); - assert_eq!(out["file_name"], "acme-w9.pdf"); - assert_eq!(std::fs::read(&to).unwrap(), std::fs::read(&src).unwrap()); + // Given inline, the file is named for its type. + assert_eq!(out["file_name"], "w9.pdf"); + assert_eq!(std::fs::read(&to).unwrap(), src); assert!(!holds_tin(out.to_string().as_bytes(), SSN_DIGITS)); #[cfg(unix)] { @@ -804,10 +836,10 @@ fn the_w9_file_goes_to_the_vault_never_the_documents() { ); // A W-9 file alone (the TIN typed in later) records the vault too. m.add_party("Solo", &["contractor"]); - let solo = m.w9_file("solo.png", "000"); + let solo = m.w9_file("000"); let Out::Ok(p) = m.w9( "Solo", - json!({"tin_last4": "0000", "w9_file_path": solo.to_str().unwrap()}), + json!({"tin_last4": "0000", "w9_content_base64": b64(&solo), "w9_sha256": sha(&solo)}), ) else { panic!("a W-9 file alone") }; @@ -815,22 +847,25 @@ fn the_w9_file_goes_to_the_vault_never_the_documents() { assert_eq!(p["w9"]["kept_at"], "wanderbooks vault"); } -/// A W-9 file is a PDF, PNG or JPEG, by its extension in any case -/// (.pdf, .png, .jpg, .jpeg): each is taken into the vault and comes back -/// byte for byte, under its own name. +/// A W-9 file is a PDF, PNG or JPEG, judged by its own first bytes: each +/// is taken into the vault and comes back byte for byte, named for its +/// type. #[test] fn every_w9_file_type_is_taken() { let mut m = books("w9-types"); - for (n, name) in ["w9.pdf", "w9.png", "w9.jpg", "w9.jpeg", "W9.JPG"] - .iter() - .enumerate() + for (n, (bytes, name)) in [ + (b"%PDF-1.4 Form W-9".as_slice(), "w9.pdf"), + (b"\x89PNG\r\n\x1a\n\0\0\0\rIHDR W-9".as_slice(), "w9.png"), + (b"\xff\xd8\xff\xe0\0\x10JFIF W-9".as_slice(), "w9.jpg"), + ] + .iter() + .enumerate() { let party = format!("Payee {n}"); m.add_party(&party, &["contractor"]); - let src = m.w9_file(name, "000"); let Out::Ok(p) = m.w9( &party, - json!({"tin_last4": "0000", "w9_file_path": src.to_str().unwrap()}), + json!({"tin_last4": "0000", "w9_content_base64": b64(bytes), "w9_sha256": sha(bytes)}), ) else { panic!("{name} refused") }; @@ -841,17 +876,57 @@ fn every_w9_file_type_is_taken() { json!({"party": party, "save_to": to.to_str().unwrap()}), ); assert_eq!(out["file_name"], *name); - assert_eq!(std::fs::read(&to).unwrap(), std::fs::read(&src).unwrap()); + assert_eq!(std::fs::read(&to).unwrap(), *bytes); } + // Its own name, when given; given alone, a name is refused. + m.add_party("Payee N", &["contractor"]); + let bytes = b"%PDF-1.4 Form W-9 named"; + let Out::Ok(_) = m.w9( + "Payee N", + json!({"tin_last4": "0000", "w9_content_base64": b64(bytes), "w9_sha256": sha(bytes), + "w9_file_name": "payee-n-w9.pdf"}), + ) else { + panic!("named W-9 refused") + }; + let to = m.outside().join("copy-named"); + let out = m.ok( + "books.get_w9", + json!({"party": "Payee N", "save_to": to.to_str().unwrap()}), + ); + assert_eq!(out["file_name"], "payee-n-w9.pdf"); + let why = m + .w9( + "Payee N", + json!({"tin_last4": "0000", "w9_file_name": "x.pdf"}), + ) + .err_text(); + assert!(why.contains("w9_file_name names the W-9"), "{why}"); + // Anything else is refused, near misses too. + m.add_party("Payee X", &["contractor"]); + for bytes in [ + b"PDF-1.4 no percent".as_slice(), + b"\x89PNG\r\n no".as_slice(), + b"\xff\xd8 short".as_slice(), + b"GIF89a".as_slice(), + ] { + let why = m + .w9( + "Payee X", + json!({"tin_last4": "0000", "w9_content_base64": b64(bytes), "w9_sha256": sha(bytes)}), + ) + .err_text(); + assert!(why.contains("PDF, PNG, or JPEG"), "{why}"); + } + assert_eq!(m.party("Payee X", 2026)["party"]["w9"], Value::Null); } #[test] fn a_locked_vault_says_so_and_never_guesses() { let mut m = books("locked"); - let src = m.w9_file("w9.pdf", SSN); + let src = m.w9_file(SSN); m.w9( "Acme Design", - json!({"tin": SSN, "tin_type": "ssn", "w9_file_path": src.to_str().unwrap()}), + json!({"tin": SSN, "tin_type": "ssn", "w9_content_base64": b64(&src), "w9_sha256": sha(&src)}), ); m.pay("Acme Design", "700.00"); let key = std::fs::read(m.key()).unwrap(); @@ -1003,10 +1078,10 @@ fn the_1099_worksheet_carries_full_tins_only_when_asked() { fn purge_deletes_the_vault_records_and_journals_why() { let mut m = books("purge"); let acme = m.party_id("Acme Design"); - let src = m.w9_file("w9.pdf", SSN); + let src = m.w9_file(SSN); m.w9( "Acme Design", - json!({"tin": SSN, "tin_type": "ssn", "w9_file_path": src.to_str().unwrap()}), + json!({"tin": SSN, "tin_type": "ssn", "w9_content_base64": b64(&src), "w9_sha256": sha(&src)}), ); // Clearing while the vault holds it: refused. let why = m @@ -1086,10 +1161,10 @@ fn no_full_tin_leaves_the_vault_except_the_worksheet_when_asked() { "no-leak", "[[company.members]]\nname = \"Mo\"\nshare = \"100%\"\ncapital_account = \"3000\"\n", ); - let src = m.w9_file("acme-w9.pdf", SSN); + let src = m.w9_file(SSN); m.w9( "Acme Design", - json!({"tin": SSN, "tin_type": "ssn", "w9_file_path": src.to_str().unwrap()}), + json!({"tin": SSN, "tin_type": "ssn", "w9_content_base64": b64(&src), "w9_sha256": sha(&src)}), ); m.w9("Rent Co", json!({"tin": EIN, "tin_type": "ein"})); m.ok("books.add_vendor", json!({"name": "Rent Co"})); diff --git a/crates/ledger-mcp/tests/startup.rs b/crates/ledger-mcp/tests/startup.rs index ca66b10..4fc016a 100644 --- a/crates/ledger-mcp/tests/startup.rs +++ b/crates/ledger-mcp/tests/startup.rs @@ -240,187 +240,25 @@ fn newer_books_are_refused_as_newer_not_corrupt() { ); } -/// Start the server and report whether it refused to start (it never got -/// as far as serving), with its standard error. -fn refuses_to_start(cmd: &mut Command) -> Option { - let mut child = cmd - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() +/// `--import-root` is gone (files come in as tool input, P2 slice 5): a +/// server started with it stops at once, as for any unknown argument, +/// and never serves as if it had read from that folder. +#[test] +fn the_import_root_flag_is_refused_as_unknown() { + let dir = temp("import-root"); + let out = Command::new(BIN) + .arg("--data-dir") + .arg(dir.join("books-data")) + .arg("--import-root") + .arg(dir.join("Downloads")) + .env("XDG_CONFIG_HOME", dir.join("config")) + .env("XDG_RUNTIME_DIR", dir.join("config")) + .stdin(Stdio::null()) + .output() .unwrap(); - // A server that starts waits on stdin; closing it ends the server. - drop(child.stdin.take()); - let out = child.wait_with_output().unwrap(); - let err = String::from_utf8_lossy(&out.stderr).into_owned(); - (!err.contains("serving over stdio")).then_some(err) -} - -/// local_files.rs INVARIANT 0: a named import root that is `/`, the home -/// folder, a folder holding it, a folder holding the books, the data dir -/// (a symlinked --data-dir placed where it really is), the vault or a key -/// file, or a folder inside the books, the data dir or the vault, stops -/// the server at startup, saying why; a folder holding none of them is -/// used. -#[test] -fn an_import_root_holding_the_books_or_the_home_folder_is_refused() { - let tmp = temp("bad-roots"); - let dir = tmp.canonicalize().unwrap(); - let home = dir.join("home"); - let books = home.join("work/books"); - let data = books.join("books-data"); - let keys = dir.join("keys"); - for d in [&home.join("Downloads"), &books, &keys, &dir.join("config")] { - std::fs::create_dir_all(d).unwrap(); - } - std::fs::write( - books.join("wanderbooks.toml"), - format!( - "[vault]\nkey_file = \"{}\"\n", - keys.join("vault.key").display() - ), - ) - .unwrap(); - let link = dir.join("data-link"); - std::os::unix::fs::symlink(&data, &link).unwrap(); - let start = |data: &std::path::Path, root: &std::path::Path| { - let mut cmd = Command::new(BIN); - cmd.arg("--data-dir") - .arg(data) - .arg("--import-root") - .arg(root) - .env("HOME", &home) - .env("XDG_CONFIG_HOME", dir.join("config")) - .env("XDG_RUNTIME_DIR", dir.join("config")); - refuses_to_start(&mut cmd) - }; - for (root, why) in [ - (std::path::PathBuf::from("/"), "whole filesystem"), - (home.clone(), "home folder"), - (dir.clone(), "home folder"), - (home.join("work"), "holds the books"), - (books.clone(), "holds the books"), - (data.clone(), "holds the books"), - ( - keys.clone(), - "holds the books, the TIN vault, or a vault key", - ), - ( - dir.join("config"), - "holds the books, the TIN vault, or a vault key", - ), - ] { - let err = start(&data, &root).unwrap_or_else(|| panic!("{} was used", root.display())); - assert!( - err.contains("can't be used") && err.contains(why), - "{}: {err}", - root.display() - ); - } - // A folder inside the books, the data dir or the vault: every file in - // it would be refused, so the root is. - for root in [ - books.join("company"), - data.join("blobs"), - books.join("vault/inner"), - ] { - std::fs::create_dir_all(&root).unwrap(); - let err = start(&data, &root).unwrap_or_else(|| panic!("{} was used", root.display())); - assert!( - err.contains("can't be used") && err.contains("inside the books"), - "{}: {err}", - root.display() - ); - } - // Through a symlinked data dir, the books are where they really are. - let err = start(&link, &home.join("work")).expect("a root holding the real books"); - assert!(err.contains("holds the books"), "{err}"); - // A folder holding none of them is used. - assert_eq!(start(&data, &home.join("Downloads")), None); -} - -/// local_files.rs INVARIANT 0, the default: an XDG_DOWNLOAD_DIR of -/// "$HOME/" (or "/") never makes the home folder an import root. With no -/// real ~/Downloads there is no root, and an import says so; with one, it -/// is the root, and the rest of the home folder stays out of reach. -#[test] -fn a_download_dir_of_home_never_opens_the_home_folder() { - let tmp = temp("xdg-home"); - let dir = tmp.canonicalize().unwrap(); - let home = dir.join("home"); - std::fs::create_dir_all(home.join(".ssh")).unwrap(); - std::fs::write(home.join(".ssh/id_ed25519"), "PRIVATE KEY").unwrap(); - let key = home.join(".ssh/id_ed25519"); - let import = |xdg: &str, path: &std::path::Path| { - one_call( - Command::new(BIN) - .arg("--data-dir") - .arg(dir.join("books/books-data")) - .env("HOME", &home) - .env("XDG_DOWNLOAD_DIR", xdg) - .env("XDG_CONFIG_HOME", dir.join("config")) - .env("XDG_RUNTIME_DIR", dir.join("config")), - "books.import_local_file", - serde_json::json!({"path": path.to_str().unwrap(), "category": "document"}), - ) - }; - for xdg in ["$HOME/", "$HOME", "/"] { - let reply = import(xdg, &key); - assert!(reply.contains("has no import folder"), "{xdg}: {reply}"); - } - std::fs::create_dir_all(home.join("Downloads")).unwrap(); - std::fs::write(home.join("Downloads/receipt.pdf"), "%PDF a receipt").unwrap(); - for xdg in ["$HOME/", "/"] { - let reply = import(xdg, &key); - assert!( - reply.contains("no such file in the server's import folders"), - "{xdg}: {reply}" - ); - assert!(!reply.contains("PRIVATE"), "{reply}"); - } - let reply = import("$HOME/", &home.join("Downloads/receipt.pdf")); - assert!(reply.contains("attachment_id"), "{reply}"); -} - -/// local_files.rs INVARIANT 0, a refused default: a downloads folder -/// that holds the books, or one inside them, is dropped, never fatal -/// (only a named root stops the server): it serves, with no import -/// folder, and an import says so. -#[test] -fn a_default_download_dir_holding_the_books_is_dropped_not_fatal() { - let tmp = temp("xdg-books"); - let dir = tmp.canonicalize().unwrap(); - let home = dir.join("home"); - let downloads = dir.join("downloads"); - std::fs::create_dir_all(&home).unwrap(); - std::fs::create_dir_all(downloads.join("books")).unwrap(); - std::fs::write(downloads.join("receipt.pdf"), "%PDF a receipt").unwrap(); - let reply = one_call( - Command::new(BIN) - .arg("--data-dir") - .arg(downloads.join("books/books-data")) - .env("HOME", &home) - .env("XDG_DOWNLOAD_DIR", &downloads) - .env("XDG_CONFIG_HOME", dir.join("config")) - .env("XDG_RUNTIME_DIR", dir.join("config")), - "books.import_local_file", - serde_json::json!({"path": downloads.join("receipt.pdf").to_str().unwrap(), "category": "document"}), - ); - assert!(reply.contains("has no import folder"), "{reply}"); - // A downloads folder inside the books dir is dropped the same way. - let books = dir.join("books"); - std::fs::create_dir_all(books.join("Downloads")).unwrap(); - std::fs::write(books.join("Downloads/receipt.pdf"), "%PDF a receipt").unwrap(); - let reply = one_call( - Command::new(BIN) - .arg("--data-dir") - .arg(books.join("books-data")) - .env("HOME", &home) - .env("XDG_DOWNLOAD_DIR", books.join("Downloads")) - .env("XDG_CONFIG_HOME", dir.join("config")) - .env("XDG_RUNTIME_DIR", dir.join("config")), - "books.import_local_file", - serde_json::json!({"path": books.join("Downloads/receipt.pdf").to_str().unwrap(), "category": "document"}), - ); - assert!(reply.contains("has no import folder"), "{reply}"); + assert_eq!(out.status.code(), Some(2)); + let err = String::from_utf8_lossy(&out.stderr); + assert!(err.contains("unknown argument: --import-root"), "{err}"); + assert!(!err.contains("serving"), "{err}"); + assert!(!dir.join("books-data").exists(), "opened the books"); } diff --git a/crates/ledger-mcp/tests/tin_files.rs b/crates/ledger-mcp/tests/tin_files.rs index f70955c..d9cb8a5 100644 --- a/crates/ledger-mcp/tests/tin_files.rs +++ b/crates/ledger-mcp/tests/tin_files.rs @@ -9,8 +9,8 @@ //! SOURCE: rules.md B-26; DECISION (P0 review round 2, 2026-10-16). //! INVARIANTS (oracle): //! 1. Every file the server read or wrote with full TINs is refused by -//! every way a file enters the books (books.import_local_file, -//! books.attach_receipt, books.file_document), from any session and +//! every way a file enters the books (books.attach_receipt and +//! books.file_document, its bytes given inline), from any session and //! after the server restarts. //! 2. Text holding a full TIN is refused, whoever made it: one the vault //! holds, in every common rendering (nine digits; 3-2-4 joined by @@ -59,8 +59,9 @@ impl Drop for Server { } } -/// The books dir, the import root, a folder outside both, and the -/// settings (a member, so the CPA package has a K-1 worksheet). +/// The books dir, a folder for files on their way in, a folder outside +/// both, and the settings (a member, so the CPA package has a K-1 +/// worksheet). struct Books { dir: PathBuf, inbox: PathBuf, @@ -125,8 +126,6 @@ fn serve(b: &Books) -> (Server, String) { .arg(b.dir.join("books-data")) .arg("--books-toml") .arg(&b.toml) - .arg("--import-root") - .arg(&b.inbox) .arg("--http") .arg(&addr) .env("XDG_CONFIG_HOME", base.join("config")) @@ -177,14 +176,17 @@ async fn ok(c: &Client, tool: &str, args: Value) -> Value { .unwrap_or_else(|e| panic!("{tool}: {e}")) } -fn b64(bytes: &[u8]) -> String { +/// `bytes` as a tool takes them inline: base64, and their SHA-256 (hex). +fn inline(bytes: &[u8]) -> (String, String) { use base64::Engine; - base64::engine::general_purpose::STANDARD.encode(bytes) + ( + base64::engine::general_purpose::STANDARD.encode(bytes), + ledger_store::journal::sha256_hex(bytes), + ) } -/// Every way the file at `path` (copied into the import root as `name`) -/// could enter the books is refused with `why`, and nothing is stored: -/// the refusals, tool by tool. +/// Every way the file at `path` (named `name`) could enter the books is +/// refused with `why`, and nothing is stored: the refusals, tool by tool. async fn refused_everywhere( c: &Client, b: &Books, @@ -193,23 +195,20 @@ async fn refused_everywhere( why: &str, ) -> Vec { let bytes = std::fs::read(path).unwrap(); - let copy = b.inbox.join(name); - std::fs::copy(path, ©).unwrap(); + let (content, sha) = inline(&bytes); let journal = b.dir.join("books-data").join(ledger_store::JOURNAL_FILE); let before = std::fs::read(&journal).unwrap(); let mut refusals = Vec::new(); for (tool, args) in [ - ( - "books.import_local_file", - json!({"path": copy.to_str().unwrap(), "category": "document", "document_category": "tax"}), - ), ( "books.attach_receipt", - json!({"file_name": name, "mime_type": "application/octet-stream", "content_base64": b64(&bytes)}), + json!({"file_name": name, "mime_type": "application/octet-stream", + "content_base64": content, "sha256": sha}), ), ( "books.file_document", - json!({"category": "tax", "file_name": name, "mime_type": "application/pdf", "content_base64": b64(&bytes)}), + json!({"category": "tax", "file_name": name, "mime_type": "application/pdf", + "content_base64": content, "sha256": sha}), ), ] { let err = call(c, tool, args).await.expect_err(tool); @@ -221,7 +220,6 @@ async fn refused_everywhere( before, "a refusal stored something" ); - std::fs::remove_file(copy).unwrap(); refusals } @@ -240,6 +238,7 @@ async fn a_tin_bearing_file_is_refused_from_every_session_and_after_a_restart() b"%PDF-1.4\n\xff\xfe\x00 scanned W-9 \x01\x02\x03\n%%EOF\n", ) .unwrap(); + let (w9_content, w9_sha) = inline(&std::fs::read(&w9).unwrap()); let (server, url) = serve(&b); let one = connect(&url).await; for (code, name, ty) in [ @@ -270,7 +269,7 @@ async fn a_tin_bearing_file_is_refused_from_every_session_and_after_a_restart() &one, "books.set_w9", json!({"party": "Acme", "received_on": "2026-01-03", "entity_type": "Individual", - "tin": SSN, "tin_type": "ssn", "w9_file_path": w9.to_str().unwrap()}), + "tin": SSN, "tin_type": "ssn", "w9_content_base64": w9_content, "w9_sha256": w9_sha}), ) .await; ok( @@ -281,7 +280,7 @@ async fn a_tin_bearing_file_is_refused_from_every_session_and_after_a_restart() .await; // A bank statement filed in the books: the package carries it, and it // is no TIN file (its routing numbers are not TINs). - accepted_everywhere(&one, &b, "statement.csv", BANK_CSV).await; + accepted_everywhere(&one, "statement.csv", BANK_CSV).await; // What the server writes with TINs, from the first session. let copy = b.out.join("w9-copy.pdf"); ok( @@ -333,12 +332,12 @@ async fn a_tin_bearing_file_is_refused_from_every_session_and_after_a_restart() std::fs::write(&fresh, format!("name,ssn\nDana,{}\n", "187-05-1907")).unwrap(); refused_everywhere(&two, &b, &fresh, "payees.csv", TIN_PRINTED).await; // An ordinary file still goes in, every way. - let fine = b.inbox.join("lunch.pdf"); - std::fs::write(&fine, b"%PDF-1.4\n\xff lunch receipt, order 12-34\n").unwrap(); + let (fine, fine_sha) = inline(b"%PDF-1.4\n\xff lunch receipt, order 12-34\n"); ok( &two, - "books.import_local_file", - json!({"path": fine.to_str().unwrap(), "category": "receipt"}), + "books.attach_receipt", + json!({"file_name": "lunch.pdf", "mime_type": "application/pdf", + "content_base64": fine, "sha256": fine_sha}), ) .await; // The books' own files the package carried are not marked: the @@ -353,7 +352,7 @@ async fn a_tin_bearing_file_is_refused_from_every_session_and_after_a_restart() "the package carries the statement" ); } - accepted_everywhere(&two, &b, "statement-again.csv", BANK_CSV).await; + accepted_everywhere(&two, "statement-again.csv", BANK_CSV).await; drop((one, two)); drop(server); @@ -406,8 +405,9 @@ async fn a_refused_set_w9_never_marks_its_file() { .await; } let w9_args = |party: &str, path: &Path| { + let (content, sha) = inline(&std::fs::read(path).unwrap()); json!({"party": party, "received_on": "2026-01-03", "entity_type": "Individual", - "tin": SSN, "tin_type": "ssn", "w9_file_path": path.to_str().unwrap()}) + "tin": SSN, "tin_type": "ssn", "w9_content_base64": content, "w9_sha256": sha}) }; for (party, path, why) in [ ("Nobody", &w9, "unknown party"), @@ -424,51 +424,54 @@ async fn a_refused_set_w9_never_marks_its_file() { ("w9-scan.pdf", w9_bytes.clone()), ("notes.txt", std::fs::read(¬es).unwrap()), ] { + let (content, sha) = inline(&bytes); let out = ok( &c, "books.file_document", json!({"category": "other", "file_name": name, "mime_type": "application/pdf", - "content_base64": b64(&bytes)}), + "content_base64": content, "sha256": sha}), ) .await; assert!(out["attachment_id"].is_u64(), "{name}: {out}"); } // A set_w9 that succeeds marks it. ok(&c, "books.set_w9", w9_args("Acme", &w9)).await; - let err = call( - &c, - "books.file_document", - json!({"category": "other", "file_name": "again.pdf", "mime_type": "application/pdf", - "content_base64": b64(&w9_bytes)}), - ) - .await - .expect_err("marked"); - assert!(err.contains(TIN_FILE), "{err}"); + let (content, sha) = inline(&w9_bytes); + for (tool, args) in [ + ( + "books.file_document", + json!({"category": "other", "file_name": "again.pdf", "mime_type": "application/pdf", + "content_base64": content, "sha256": sha}), + ), + ( + "books.attach_receipt", + json!({"file_name": "again.pdf", "mime_type": "application/pdf", + "content_base64": content, "sha256": sha}), + ), + ] { + let err = call(&c, tool, args).await.expect_err("marked"); + assert!(err.contains(TIN_FILE), "{tool}: {err}"); + } } /// Every way a file enters the books takes the text `name` holding /// `text`, and the journal shows it. -async fn accepted_everywhere(c: &Client, b: &Books, name: &str, text: &str) { - let path = b.inbox.join(name); - std::fs::write(&path, text).unwrap(); +async fn accepted_everywhere(c: &Client, name: &str, text: &str) { + let (content, sha) = inline(text.as_bytes()); for (tool, args) in [ - ( - "books.import_local_file", - json!({"path": path.to_str().unwrap(), "category": "document", "document_category": "statement"}), - ), ( "books.attach_receipt", - json!({"file_name": name, "mime_type": "text/csv", "content_base64": b64(text.as_bytes())}), + json!({"file_name": name, "mime_type": "text/csv", "content_base64": content, "sha256": sha}), ), ( "books.file_document", - json!({"category": "import_source", "file_name": name, "mime_type": "text/csv", "content_base64": b64(text.as_bytes())}), + json!({"category": "import_source", "file_name": name, "mime_type": "text/csv", + "content_base64": content, "sha256": sha}), ), ] { let out = ok(c, tool, args).await; assert!(out["attachment_id"].is_u64(), "{tool}: {out}"); } - std::fs::remove_file(path).unwrap(); } /// A real bank export: ABA routing numbers (nine digits), account and @@ -526,7 +529,7 @@ async fn text_with_a_vault_tin_or_a_printed_one_is_refused_and_a_bank_export_is_ std::fs::write(&path, format!("payee,tin\n{printed}\n")).unwrap(); refused_everywhere(&c, &b, &path, &format!("payees-{n}.csv"), TIN_PRINTED).await; } - accepted_everywhere(&c, &b, "bank.csv", BANK_CSV).await; + accepted_everywhere(&c, "bank.csv", BANK_CSV).await; drop(c); drop(server); @@ -542,12 +545,11 @@ async fn text_with_a_vault_tin_or_a_printed_one_is_refused_and_a_bank_export_is_ refused_everywhere(&c, &b, &path, "members-1.csv", TIN_PRINTED).await; accepted_everywhere( &c, - &b, "bank-2.csv", &format!("{BANK_CSV}2026-10-06,x,1.00,1.00,1\n"), ) .await; - accepted_everywhere(&c, &b, "members-0.csv", "member,tin\nKat,830835531\n").await; + accepted_everywhere(&c, "members-0.csv", "member,tin\nKat,830835531\n").await; } /// INVARIANT 3b through the tools: the company's own EIN, as its @@ -571,7 +573,6 @@ async fn the_company_s_own_ein_goes_in_and_no_other() { let c = connect(&url).await; accepted_everywhere( &c, - &b, "plan.md", "# Tin Co business plan\n\nEIN 12-3456789, formed 2026-10-04.\n", ) @@ -697,8 +698,8 @@ async fn text_probing_for_a_tin_is_refused_alike_by_every_filing_tool() { }; let open = probes(&c).await; refused_everywhere(&c, &b, &tin, "tin.csv", TIN_HELD).await; - accepted_everywhere(&c, &b, "one.csv", &std::fs::read_to_string(&one).unwrap()).await; - accepted_everywhere(&c, &b, "bank.csv", BANK_CSV).await; + accepted_everywhere(&c, "one.csv", &std::fs::read_to_string(&one).unwrap()).await; + accepted_everywhere(&c, "bank.csv", BANK_CSV).await; drop(c); drop(server); diff --git a/crates/ledger-store/src/auth.rs b/crates/ledger-store/src/auth.rs index a7e1e08..4b7324b 100644 --- a/crates/ledger-store/src/auth.rs +++ b/crates/ledger-store/src/auth.rs @@ -168,6 +168,10 @@ impl From for AuthError { } } +/// A token's name when none is given (`wanderbooks token create`, the +/// bridge, `upload`, `download`). +pub const DEFAULT_NAME: &str = "agent"; + /// Check a token's name: 1 to 32 lowercase letters, digits and hyphens, /// starting with a letter or digit. pub fn check_name(name: &str) -> Result<(), AuthError> { diff --git a/crates/ledger-store/src/lib.rs b/crates/ledger-store/src/lib.rs index 41875ee..f69754e 100644 --- a/crates/ledger-store/src/lib.rs +++ b/crates/ledger-store/src/lib.rs @@ -32,6 +32,7 @@ pub mod root; pub mod schema; pub mod snapshot; pub mod store; +pub mod uploads; pub mod viewer; pub use blob::BlobStore; diff --git a/crates/ledger-store/src/uploads.rs b/crates/ledger-store/src/uploads.rs new file mode 100644 index 0000000..c90509e --- /dev/null +++ b/crates/ledger-store/src/uploads.rs @@ -0,0 +1,1045 @@ +//! Files on their way in (plan 4.7, P2 slice 5): an agent asks for an +//! upload (`books.begin_upload`), sends the bytes on its own (`wanderbooks +//! upload`, which streams them to the server), then names the upload in +//! the tool that files them. +//! +//! ```text +//! GOAL: a file of any size up to the limit reaches the books whole, +//! exactly as declared, without passing through the agent's context +//! or naming a path on the server. +//! SOURCE: plan 4.7 ("single-use, valid 15 minutes, bound to the +//! principal, sha and size, capped at 25 MiB by default, and +//! re-hashed before acceptance"), D-28 (the agent does it). +//! INVARIANTS: +//! 1. A grant is made for one token, one SHA-256 and one size, and +//! lasts UPLOAD_TTL; its bytes are received once, by that token +//! alone, and filed once: a call that takes them holds them (no other +//! call can) and spends them only when it files them; a call that +//! fails, or panics, gives them back for the next. +//! 2. Bytes are written aside (`.part`, by one request at a time) +//! and kept (`.bin`) only if they are exactly the size declared +//! and hash to the SHA-256 declared; anything else leaves nothing but +//! a refusal. Taken, they are hashed again. +//! 3. Grants and their bytes live in the book's `uploads/` folder (0700, +//! files 0600), so they survive a restart; lapsed ones are swept when +//! the books open, whenever a grant is made, and (by the server) every +//! 10 seconds, so none lingers; at open, bytes a call held when the +//! server stopped are given back. +//! 4. No error names a path. +//! 5. A token has at most MAX_LIVE grants under way, so what one agent +//! can keep on the disk is bounded. +//! ``` + +use crate::journal::sha256_hex; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; + +/// The uploads' folder, in a book's folder. +pub const UPLOADS_DIR: &str = "uploads"; +/// The largest file taken (plan 4.7). +pub const MAX_UPLOAD: u64 = 25 * 1024 * 1024; +/// How long a grant lasts, seconds (plan 4.7: 15 minutes). +pub const UPLOAD_TTL: i64 = 15 * 60; +/// The most grants one token may have under way at once (INVARIANT 5). +pub const MAX_LIVE: usize = 16; +/// What an upload may be for: the tools that file it take only theirs. +pub const PURPOSES: [&str; 4] = ["receipt", "document", "import", "w9"]; + +/// A grant: what may be sent, by whom, until when (INVARIANT 1). +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Grant { + /// Its id: `up_` and 32 letters `a` to `p`. + pub id: String, + /// The token it was made for. + pub token: String, + /// The bytes' SHA-256, hex. + pub sha256: String, + /// Their size. + pub size: u64, + /// Their MIME type. + pub mime: String, + /// The file's name. + pub file_name: String, + /// What it is for: receipt, document, import (a CSV) or w9. + pub purpose: String, + /// When it lapses (Unix seconds). + pub expires: i64, +} + +/// Bytes received and checked, ready to file. +#[derive(Debug)] +pub struct Received { + /// The grant they came under. + pub grant: Grant, + /// The bytes. + pub bytes: Vec, + /// The hold on them: spend it once they're filed (INVARIANT 1). + pub claim: Claim, +} + +/// A call's hold on an upload's bytes (INVARIANT 1): [`Claim::spend`] +/// once they're filed; dropped unspent, they're given back. +#[derive(Debug)] +pub struct Claim { + uploads: Uploads, + id: String, + spent: bool, +} + +impl Claim { + /// The bytes are filed: the upload is used up. + pub fn spend(mut self) { + self.spent = true; + self.uploads.forget(&self.id); + } +} + +impl Drop for Claim { + fn drop(&mut self) { + if !self.spent { + let _held = self.uploads.locked(); + let _ = std::fs::rename( + self.uploads.path(&self.id, TAKEN), + self.uploads.path(&self.id, "bin"), + ); + } + } +} + +/// Why an upload is refused. Never names a path (INVARIANT 4). +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum UploadError { + /// No such grant, or it lapsed. + #[error("there is no upload {0:?}, or it lapsed (an upload lasts 15 minutes): begin another")] + Unknown(String), + /// Another token's. + #[error("that upload was begun with another token")] + NotYours, + /// Its bytes came already. + #[error("that upload's bytes were sent already")] + AlreadySent, + /// Another request is sending its bytes, or another call is filing + /// them. + #[error("that upload is in use by another request: wait for it to finish")] + InUse, + /// Its bytes haven't come. + #[error( + "that upload's bytes haven't been sent yet: send them with the command books.begin_upload gave" + )] + NotSent, + /// Not the size declared. + #[error("the bytes sent were {got} long, not the {want} declared: nothing was kept")] + WrongSize { + /// Declared. + want: u64, + /// Sent. + got: u64, + }, + /// More than the size declared. + #[error("more than the {want} bytes declared were sent: nothing was kept")] + TooMuch { + /// Declared. + want: u64, + }, + /// Not the SHA-256 declared. + #[error("the bytes sent don't have the SHA-256 declared: nothing was kept")] + WrongHash, + /// Over the limit. + #[error("a file is at most {MAX_UPLOAD} bytes")] + TooLarge, + /// A request that can't make a grant. + #[error("{0}")] + Bad(String), + /// The disk refused. + #[error("the upload couldn't be kept: {0}")] + Io(std::io::ErrorKind), +} + +impl From for UploadError { + fn from(e: std::io::Error) -> Self { + UploadError::Io(e.kind()) + } +} + +/// A book's uploads. +#[derive(Debug, Clone)] +pub struct Uploads { + dir: PathBuf, + /// Held while grants are made or swept and while bytes change name + /// (INVARIANTS 1, 2, 5): one server serves a book, so this orders + /// every request to its uploads. + lock: std::sync::Arc>, +} + +/// The longest file name taken, in bytes (most file systems' limit). +pub const MAX_NAME_BYTES: usize = 255; + +/// Characters that change how a name shows without showing themselves +/// (Unicode's format characters: bidirectional overrides, zero widths). +pub fn is_hidden_char(c: char) -> bool { + matches!(c, + // Format characters (Unicode's Cf). + '\u{00AD}' | '\u{0600}'..='\u{0605}' | '\u{061C}' | '\u{06DD}' + | '\u{070F}' | '\u{0890}' | '\u{0891}' | '\u{08E2}' | '\u{180E}' + | '\u{200B}'..='\u{200F}' + | '\u{202A}'..='\u{202E}' | '\u{2060}'..='\u{2064}' + | '\u{2066}'..='\u{206F}' | '\u{FEFF}' | '\u{FFF9}'..='\u{FFFB}' + | '\u{110BD}' | '\u{110CD}' | '\u{13430}'..='\u{1343F}' + | '\u{1BCA0}'..='\u{1BCA3}' | '\u{1D173}'..='\u{1D17A}' + | '\u{E0001}' | '\u{E0020}'..='\u{E007F}' + // Line and paragraph separators (Zl, Zp). + | '\u{2028}' | '\u{2029}' + // Characters that show as nothing: the grapheme joiner, the Hangul + // fillers, the Khmer inherent vowels, the variation selectors. + | '\u{034F}' | '\u{115F}' | '\u{1160}' | '\u{17B4}' | '\u{17B5}' + | '\u{180B}'..='\u{180F}' | '\u{2800}' | '\u{3164}' | '\u{FFA0}' + | '\u{FE00}'..='\u{FE0F}' | '\u{E0100}'..='\u{E01EF}') +} + +/// Why `name` can't name a file filed in the books, if it can't: a plain +/// name of 1 to MAX_NAME_BYTES bytes, no folder in it, nothing invisible. +pub fn file_name_problem(name: &str) -> Option { + if name.trim().is_empty() { + return Some("a file name is needed".into()); + } + if name.len() > MAX_NAME_BYTES { + return Some(format!("a file name is at most {MAX_NAME_BYTES} bytes")); + } + if name.contains(['/', '\\']) || name == "." || name == ".." { + return Some("a file name is a name alone, with no folder in it".into()); + } + if name.chars().any(|c| c.is_control() || is_hidden_char(c)) { + return Some("a file name has no control or invisible characters".into()); + } + None +} + +/// Why `mime` can't be a file's type, if it can't: `type/subtype` (and +/// any parameters), visible ASCII, at most 127 bytes. +pub fn mime_problem(mime: &str) -> Option { + let ok = (3..=127).contains(&mime.len()) + && mime.bytes().all(|b| b.is_ascii_graphic() || b == b' ') + && mime + .split(';') + .next() + .and_then(|t| t.trim().split_once('/')) + .is_some_and(|(a, b)| !a.is_empty() && !b.is_empty()); + (!ok).then(|| "a file's type is a MIME type such as application/pdf".to_string()) +} + +/// The bytes of an upload a call holds (INVARIANT 1). +const TAKEN: &str = "taken"; + +/// An upload's id: `up_` and 32 letters `a` to `p` (16 random bytes, a +/// letter a nibble). No digits, so the access log, which masks what could +/// be a TIN, keeps an id whole. +pub fn is_id(id: &str) -> bool { + id.len() == 3 + 32 + && id.starts_with("up_") + && id[3..].bytes().all(|b| (b'a'..=b'p').contains(&b)) +} + +fn new_id(raw: &[u8; 16]) -> String { + let mut id = String::from("up_"); + for b in raw { + id.push(char::from(b'a' + (b >> 4))); + id.push(char::from(b'a' + (b & 0x0f))); + } + id +} + +fn private_file(path: &Path) -> std::io::Result { + let mut o = std::fs::OpenOptions::new(); + o.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + o.mode(0o600); + } + o.open(path) +} + +impl Uploads { + /// The uploads of the book in `book_dir` (its folder made, private, if + /// missing), as the books open, before any call can run (INVARIANT 3): + /// lapsed grants swept, bytes a call held given back, and any half + /// sent thrown away. + pub fn open(book_dir: &Path, now: i64) -> Result { + let dir = crate::root::private_book_folder(book_dir, UPLOADS_DIR) + .map_err(|e| UploadError::Bad(format!("the uploads can't be kept: {e}")))?; + let uploads = Self { + dir, + lock: Default::default(), + }; + if let Ok(entries) = std::fs::read_dir(&uploads.dir) { + for e in entries.flatten() { + let name = e.file_name().to_string_lossy().to_string(); + if let Some(id) = name.strip_suffix(&format!(".{TAKEN}")) { + let _ = std::fs::rename(e.path(), uploads.path(id, "bin")); + } else if name.ends_with(".part") { + let _ = std::fs::remove_file(e.path()); + } + } + } + uploads.sweep(now); + Ok(uploads) + } + + fn locked(&self) -> std::sync::MutexGuard<'_, ()> { + self.lock.lock().unwrap_or_else(|e| e.into_inner()) + } + + fn path(&self, id: &str, ext: &str) -> PathBuf { + self.dir.join(format!("{id}.{ext}")) + } + + /// Make a grant for `token` (INVARIANT 1). + #[allow(clippy::too_many_arguments)] + pub fn grant( + &self, + token: &str, + sha256: &str, + size: u64, + mime: &str, + file_name: &str, + purpose: &str, + now: i64, + ) -> Result { + let sha = sha256.trim().to_ascii_lowercase(); + if sha.len() != 64 || !sha.bytes().all(|b| b.is_ascii_hexdigit()) { + return Err(UploadError::Bad( + "sha256 is the file's SHA-256: 64 hex digits".into(), + )); + } + if size == 0 { + return Err(UploadError::Bad("an empty file isn't filed".into())); + } + if let Some(why) = file_name_problem(file_name).or_else(|| mime_problem(mime)) { + return Err(UploadError::Bad(why)); + } + if size > MAX_UPLOAD { + return Err(UploadError::TooLarge); + } + if !PURPOSES.contains(&purpose) { + return Err(UploadError::Bad( + "purpose is receipt, document, import (a bank or chart CSV) or w9".into(), + )); + } + let _held = self.locked(); + self.sweep_held(now); + // INVARIANT 5. + if self.live_for(token, now) >= MAX_LIVE { + return Err(UploadError::Bad(format!( + "this agent has {MAX_LIVE} uploads under way: file them, or let them lapse (15 minutes)" + ))); + } + let mut raw = [0u8; 16]; + getrandom::getrandom(&mut raw) + .map_err(|_| UploadError::Bad("the system gave no randomness".into()))?; + let grant = Grant { + id: new_id(&raw), + token: token.to_string(), + sha256: sha, + size, + mime: mime.to_string(), + file_name: file_name.to_string(), + purpose: purpose.to_string(), + expires: now + UPLOAD_TTL, + }; + // Written aside and renamed, so a grant is read whole or not at all. + let aside = self.path(&grant.id, "json.new"); + let mut f = private_file(&aside)?; + f.write_all(&serde_json::to_vec(&grant).expect("a grant serializes"))?; + f.sync_all()?; + drop(f); + std::fs::rename(&aside, self.path(&grant.id, "json"))?; + Ok(grant) + } + + /// How many grants `token` has under way at `now`. + fn live_for(&self, token: &str, now: i64) -> usize { + let Ok(entries) = std::fs::read_dir(&self.dir) else { + return 0; + }; + entries + .flatten() + .filter(|e| e.file_name().to_string_lossy().ends_with(".json")) + .filter_map(|e| std::fs::read(e.path()).ok()) + .filter_map(|b| serde_json::from_slice::(&b).ok()) + .filter(|g| g.token == token && g.expires > now) + .count() + } + + /// The live grant `id`, for `token` (INVARIANT 1). + fn live(&self, id: &str, token: &str, now: i64) -> Result { + if !is_id(id) { + return Err(UploadError::Unknown(id.chars().take(40).collect())); + } + let unknown = || UploadError::Unknown(id.to_string()); + let bytes = std::fs::read(self.path(id, "json")).map_err(|_| unknown())?; + let grant: Grant = serde_json::from_slice(&bytes).map_err(|_| unknown())?; + if grant.expires <= now { + self.forget(id); + return Err(unknown()); + } + if grant.token != token { + return Err(UploadError::NotYours); + } + Ok(grant) + } + + /// Receive grant `id`'s bytes from `body`, by `token` (INVARIANTS 1, + /// 2): kept only if exactly as declared. + pub fn receive( + &self, + id: &str, + token: &str, + body: &mut impl Read, + now: i64, + ) -> Result { + let grant = self.live(id, token, now)?; + let started = std::time::Instant::now(); + let done = self.path(id, "bin"); + let part = self.path(id, "part"); + // One request at a time (INVARIANT 2): no bytes kept or held, and + // no `.part` of another request. + let mut out = { + let _held = self.locked(); + if done.exists() || self.path(id, TAKEN).exists() { + return Err(UploadError::AlreadySent); + } + match private_file(&part) { + Ok(f) => f, + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { + return Err(UploadError::InUse); + } + Err(e) => return Err(e.into()), + } + }; + let mut hasher = Sha256::new(); + let mut got: u64 = 0; + let mut buf = vec![0u8; 64 * 1024]; + // Read one byte past the size declared, to know when there's more. + let mut limited = body.take(grant.size + 1); + let read = loop { + let n = match limited.read(&mut buf) { + Ok(0) => break Ok(()), + Ok(n) => n, + Err(e) => break Err(UploadError::from(e)), + }; + got += n as u64; + if got > grant.size { + break Ok(()); + } + hasher.update(&buf[..n]); + if let Err(e) = out.write_all(&buf[..n]) { + break Err(e.into()); + } + }; + let checked = read.and_then(|()| { + if got > grant.size { + return Err(UploadError::TooMuch { want: grant.size }); + } + if got != grant.size { + return Err(UploadError::WrongSize { + want: grant.size, + got, + }); + } + if hex::encode(hasher.finalize()) != grant.sha256 { + return Err(UploadError::WrongHash); + } + out.sync_all()?; + Ok(()) + }); + drop(out); + match checked { + Ok(()) => { + // Kept only if its grant still stands (it may have lapsed, + // or been filed by another request's bytes, meanwhile). + let _held = self.locked(); + let standing = std::fs::read(self.path(id, "json")) + .ok() + .and_then(|b| serde_json::from_slice::(&b).ok()) + // At the time the bytes finished coming, not began. + .is_some_and(|g| { + g == grant && g.expires > now + started.elapsed().as_secs() as i64 + }); + if !standing || self.path(id, TAKEN).exists() || done.exists() { + let _ = std::fs::remove_file(&part); + return Err(if standing { + UploadError::AlreadySent + } else { + UploadError::Unknown(id.to_string()) + }); + } + std::fs::rename(&part, &done)?; + Ok(grant) + } + Err(e) => { + let _ = std::fs::remove_file(&part); + Err(e) + } + } + } + + /// Take grant `id`'s bytes, by `token` (INVARIANTS 1, 2): held for + /// this call alone, checked again against the SHA-256 declared, and + /// used up only when the claim is spent. + pub fn take(&self, id: &str, token: &str, now: i64) -> Result { + let grant = self.live(id, token, now)?; + let taken = self.path(id, TAKEN); + // The rename is the hold: of two calls, one gets it. + { + let _held = self.locked(); + if let Err(e) = std::fs::rename(self.path(id, "bin"), &taken) { + return Err(match e.kind() { + std::io::ErrorKind::NotFound if taken.exists() => UploadError::InUse, + std::io::ErrorKind::NotFound => UploadError::NotSent, + k => UploadError::Io(k), + }); + } + } + let claim = Claim { + uploads: self.clone(), + id: id.to_string(), + spent: false, + }; + let bytes = match std::fs::read(&taken) { + Ok(b) => b, + Err(e) => return Err(UploadError::Io(e.kind())), + }; + if bytes.len() as u64 != grant.size || sha256_hex(&bytes) != grant.sha256 { + // Changed since it came: never filed, never given back. + claim.spend(); + return Err(UploadError::WrongHash); + } + Ok(Received { + grant, + bytes, + claim, + }) + } + + /// Throw away the bytes received for grant `id` (their record couldn't + /// be written): the grant stands, its bytes are to be sent again. + pub fn discard(&self, id: &str) { + if is_id(id) { + let _held = self.locked(); + let _ = std::fs::remove_file(self.path(id, "bin")); + } + } + + /// Forget grant `id` and anything kept for it. + fn forget(&self, id: &str) { + let _held = self.locked(); + for ext in ["json", "part", "bin", TAKEN] { + let _ = std::fs::remove_file(self.path(id, ext)); + } + } + + /// Sweep away every grant that lapsed by `now`, and anything left + /// without one (INVARIANT 3). + pub fn sweep(&self, now: i64) { + let _held = self.locked(); + self.sweep_held(now); + } + + /// [`Self::sweep`], the lock held. + fn sweep_held(&self, now: i64) { + let Ok(entries) = std::fs::read_dir(&self.dir) else { + return; + }; + for e in entries.flatten() { + let name = e.file_name().to_string_lossy().to_string(); + let Some((id, _)) = name.split_once('.') else { + let _ = std::fs::remove_file(e.path()); + continue; + }; + let lapsed = std::fs::read(self.path(id, "json")) + .ok() + .and_then(|b| serde_json::from_slice::(&b).ok()) + .is_none_or(|g| g.expires <= now); + if lapsed { + let _ = std::fs::remove_file(e.path()); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const NOW: i64 = 1_800_000_000; + + fn grant_for(u: &Uploads, token: &str, bytes: &[u8]) -> Grant { + u.grant( + token, + &sha256_hex(bytes), + bytes.len() as u64, + "application/pdf", + "r.pdf", + "receipt", + NOW, + ) + .unwrap() + } + + #[test] + fn bytes_sent_as_declared_are_taken_once() { + let dir = tempfile::tempdir().unwrap(); + let u = Uploads::open(dir.path(), NOW).unwrap(); + let bytes = b"%PDF-1.4 a receipt".to_vec(); + let g = grant_for(&u, "tok_a", &bytes); + assert!(is_id(&g.id)); + assert!(!g.id[3..].bytes().any(|b| b.is_ascii_digit()), "{}", g.id); + assert_eq!(new_id(&[0x00; 16]), format!("up_{}", "a".repeat(32))); + assert_eq!(new_id(&[0xff; 16]), format!("up_{}", "p".repeat(32))); + assert!(!is_id(&format!("up_{}", "0".repeat(32)))); + assert!(!is_id(&format!("up_{}", "q".repeat(32)))); + assert_eq!(g.expires, NOW + UPLOAD_TTL); + assert_eq!( + u.take(&g.id, "tok_a", NOW).unwrap_err(), + UploadError::NotSent + ); + u.receive(&g.id, "tok_a", &mut &bytes[..], NOW).unwrap(); + assert_eq!( + u.receive(&g.id, "tok_a", &mut &bytes[..], NOW).unwrap_err(), + UploadError::AlreadySent + ); + assert_eq!( + u.take(&g.id, "tok_b", NOW).unwrap_err(), + UploadError::NotYours + ); + let r = u.take(&g.id, "tok_a", NOW + 1).unwrap(); + assert_eq!(r.bytes, bytes); + assert_eq!(r.grant, g); + r.claim.spend(); + assert!(matches!( + u.take(&g.id, "tok_a", NOW + 2).unwrap_err(), + UploadError::Unknown(_) + )); + assert_eq!( + std::fs::read_dir(dir.path().join(UPLOADS_DIR)) + .unwrap() + .count(), + 0 + ); + } + + #[test] + fn anything_but_the_bytes_declared_leaves_nothing() { + let dir = tempfile::tempdir().unwrap(); + let u = Uploads::open(dir.path(), NOW).unwrap(); + let bytes = b"exactly these".to_vec(); + for (sent, want) in [ + (b"exactly thes".to_vec(), "short"), + (b"exactly these!".to_vec(), "long"), + (b"exactly thosE".to_vec(), "hash"), + ] { + let g = grant_for(&u, "tok_a", &bytes); + let e = u.receive(&g.id, "tok_a", &mut &sent[..], NOW).unwrap_err(); + match want { + "hash" => assert_eq!(e, UploadError::WrongHash), + "long" => assert_eq!(e, UploadError::TooMuch { want: 13 }), + _ => assert!(matches!(e, UploadError::WrongSize { .. }), "{e}"), + } + assert!( + !dir.path() + .join(UPLOADS_DIR) + .join(format!("{}.bin", g.id)) + .exists() + ); + assert!( + !dir.path() + .join(UPLOADS_DIR) + .join(format!("{}.part", g.id)) + .exists() + ); + assert_eq!( + u.take(&g.id, "tok_a", NOW).unwrap_err(), + UploadError::NotSent + ); + } + // Another token's grant takes no bytes. + let g = grant_for(&u, "tok_a", &bytes); + assert_eq!( + u.receive(&g.id, "tok_b", &mut &bytes[..], NOW).unwrap_err(), + UploadError::NotYours + ); + } + + #[test] + fn a_grant_lapses_and_is_swept() { + let dir = tempfile::tempdir().unwrap(); + let u = Uploads::open(dir.path(), NOW).unwrap(); + let bytes = b"late".to_vec(); + let g = grant_for(&u, "tok_a", &bytes); + let later = NOW + UPLOAD_TTL; + assert!(matches!( + u.receive(&g.id, "tok_a", &mut &bytes[..], later) + .unwrap_err(), + UploadError::Unknown(_) + )); + let g2 = grant_for(&u, "tok_a", &bytes); + u.receive(&g2.id, "tok_a", &mut &bytes[..], NOW).unwrap(); + std::fs::write(dir.path().join(UPLOADS_DIR).join("stray"), "x").unwrap(); + u.sweep(NOW + UPLOAD_TTL - 1); + assert!( + dir.path() + .join(UPLOADS_DIR) + .join(format!("{}.bin", g2.id)) + .exists() + ); + assert!(!dir.path().join(UPLOADS_DIR).join("stray").exists()); + u.sweep(later); + assert_eq!( + std::fs::read_dir(dir.path().join(UPLOADS_DIR)) + .unwrap() + .count(), + 0 + ); + } + + #[test] + fn a_grant_needs_a_hash_a_size_and_a_purpose() { + let dir = tempfile::tempdir().unwrap(); + let u = Uploads::open(dir.path(), NOW).unwrap(); + let sha = sha256_hex(b"x"); + for (s, size, purpose) in [ + ("nothex", 1, "receipt"), + (&sha[..63], 1, "receipt"), + (&sha, 0, "receipt"), + (&sha, MAX_UPLOAD + 1, "receipt"), + (&sha, 1, "anything"), + ] { + assert!( + u.grant("t", s, size, "text/plain", "x", purpose, NOW) + .is_err(), + "{s} {size} {purpose}" + ); + } + assert!( + u.grant( + "t", + &sha.to_uppercase(), + MAX_UPLOAD, + "text/plain", + "x", + "import", + NOW + ) + .is_ok() + ); + // An id that isn't one is no path into the folder. + for bad in ["../../journal", "up_", "up_XYZ", ""] { + assert!(matches!( + u.take(bad, "t", NOW).unwrap_err(), + UploadError::Unknown(_) + )); + } + } + + #[test] + fn bytes_are_held_by_one_call_and_given_back_unless_filed() { + let dir = tempfile::tempdir().unwrap(); + let u = Uploads::open(dir.path(), NOW).unwrap(); + let bytes = b"%PDF-1.4 held".to_vec(); + let g = grant_for(&u, "tok_a", &bytes); + u.receive(&g.id, "tok_a", &mut &bytes[..], NOW).unwrap(); + // Held: no other call gets them, and they can't be sent again. + let first = u.take(&g.id, "tok_a", NOW).unwrap(); + assert_eq!(u.take(&g.id, "tok_a", NOW).unwrap_err(), UploadError::InUse); + assert_eq!( + u.receive(&g.id, "tok_a", &mut &bytes[..], NOW).unwrap_err(), + UploadError::AlreadySent + ); + // A call that failed gives them back. + drop(first); + let again = u.take(&g.id, "tok_a", NOW).unwrap(); + assert_eq!(again.bytes, bytes); + // A call that panicked does too. + let caught = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _held = again; + panic!("the call failed"); + })); + assert!(caught.is_err()); + let filed = u.take(&g.id, "tok_a", NOW).unwrap(); + filed.claim.spend(); + assert!(matches!( + u.take(&g.id, "tok_a", NOW).unwrap_err(), + UploadError::Unknown(_) + )); + assert_eq!( + std::fs::read_dir(dir.path().join(UPLOADS_DIR)) + .unwrap() + .count(), + 0 + ); + } + + #[test] + fn opening_gives_back_what_a_stopped_server_held_and_drops_half_sends() { + let dir = tempfile::tempdir().unwrap(); + let u = Uploads::open(dir.path(), NOW).unwrap(); + let bytes = b"%PDF-1.4 kept".to_vec(); + let g = grant_for(&u, "tok_a", &bytes); + u.receive(&g.id, "tok_a", &mut &bytes[..], NOW).unwrap(); + let held = u.take(&g.id, "tok_a", NOW).unwrap(); + // The server stops mid-call: nothing runs the claim's drop. + std::mem::forget(held); + let half = grant_for(&u, "tok_a", &bytes); + let folder = dir.path().join(UPLOADS_DIR); + std::fs::write(folder.join(format!("{}.part", half.id)), b"%PDF").unwrap(); + // While running, a half send blocks another. + assert_eq!( + u.receive(&half.id, "tok_a", &mut &bytes[..], NOW) + .unwrap_err(), + UploadError::InUse + ); + // A grant that lapses by the next open is swept by it. + let lapsing = u + .grant( + "tok_a", + &sha256_hex(b"z"), + 1, + "text/plain", + "z", + "receipt", + NOW + 1 - UPLOAD_TTL, + ) + .unwrap(); + assert!(folder.join(format!("{}.json", lapsing.id)).exists()); + let u = Uploads::open(dir.path(), NOW + 1).unwrap(); + assert!(!folder.join(format!("{}.json", lapsing.id)).exists()); + assert!(!folder.join(format!("{}.part", half.id)).exists()); + u.receive(&half.id, "tok_a", &mut &bytes[..], NOW + 1) + .unwrap(); + let back = u.take(&g.id, "tok_a", NOW + 1).unwrap(); + assert_eq!(back.bytes, bytes); + back.claim.spend(); + } + + /// A body that, half read, has the grant forgotten (filed by another + /// request's bytes, or swept) under it. + struct Forgetting<'a> { + bytes: &'a [u8], + then: Option>, + } + + impl std::io::Read for Forgetting<'_> { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + if let Some(f) = self.then.take() { + f(); + } + self.bytes.read(buf) + } + } + + #[test] + fn bytes_are_kept_only_while_their_grant_stands() { + let dir = tempfile::tempdir().unwrap(); + let u = Uploads::open(dir.path(), NOW).unwrap(); + let bytes = b"%PDF-1.4 late".to_vec(); + let g = grant_for(&u, "tok_a", &bytes); + let folder = dir.path().join(UPLOADS_DIR); + let json = folder.join(format!("{}.json", g.id)); + let mut body = Forgetting { + bytes: &bytes, + then: Some(Box::new(|| std::fs::remove_file(&json).unwrap())), + }; + let e = u.receive(&g.id, "tok_a", &mut body, NOW).unwrap_err(); + assert!(matches!(e, UploadError::Unknown(_)), "{e}"); + assert_eq!( + std::fs::read_dir(&folder).unwrap().count(), + 0, + "nothing kept" + ); + // Bytes whose record couldn't be written are thrown away; the + // grant stands, for them to be sent again. + let g = grant_for(&u, "tok_a", &bytes); + u.receive(&g.id, "tok_a", &mut &bytes[..], NOW).unwrap(); + u.discard(&g.id); + assert_eq!( + u.take(&g.id, "tok_a", NOW).unwrap_err(), + UploadError::NotSent + ); + u.receive(&g.id, "tok_a", &mut &bytes[..], NOW).unwrap(); + u.discard("../../journal"); + assert_eq!(u.take(&g.id, "tok_a", NOW).unwrap().bytes, bytes); + } + + /// A body that takes its time. + struct Slow<'a>(&'a [u8], std::time::Duration); + + impl std::io::Read for Slow<'_> { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + std::thread::sleep(std::mem::take(&mut self.1)); + self.0.read(buf) + } + } + + #[test] + fn bytes_that_finish_coming_after_their_grant_lapsed_are_not_kept() { + let dir = tempfile::tempdir().unwrap(); + let u = Uploads::open(dir.path(), NOW).unwrap(); + let bytes = b"%PDF-1.4 slow".to_vec(); + // It lapses a second after the send begins; the send takes longer. + let g = u + .grant( + "tok_a", + &sha256_hex(&bytes), + bytes.len() as u64, + "application/pdf", + "r.pdf", + "receipt", + NOW + 1 - UPLOAD_TTL, + ) + .unwrap(); + let mut body = Slow(&bytes, std::time::Duration::from_millis(1100)); + let e = u.receive(&g.id, "tok_a", &mut body, NOW).unwrap_err(); + assert!(matches!(e, UploadError::Unknown(_)), "{e}"); + let folder = dir.path().join(UPLOADS_DIR); + assert!(!folder.join(format!("{}.bin", g.id)).exists()); + assert!(!folder.join(format!("{}.part", g.id)).exists()); + } + + #[test] + fn a_name_is_plain_and_visible() { + for ok in ["r.pdf", "Résumé 2026 (final).pdf", "收据.png", "a"] { + assert_eq!(file_name_problem(ok), None, "{ok}"); + } + for bad in [ + "", + " ", + ".", + "..", + "a/b", + "a\\b", + "a\nb", + "a\u{202E}b", + "a\u{200B}b", + "a\u{E0041}b", + "a\u{2028}b", + "a\u{FE0F}b", + "a\u{3164}b", + "a\u{2800}b", + "a\u{0890}b", + ] { + assert!(file_name_problem(bad).is_some(), "{bad:?}"); + } + assert_eq!(file_name_problem(&"a".repeat(MAX_NAME_BYTES)), None); + assert!(file_name_problem(&"a".repeat(MAX_NAME_BYTES + 1)).is_some()); + for ok in ["application/pdf", "text/plain; charset=utf-8", "a/b"] { + assert_eq!(mime_problem(ok), None, "{ok}"); + } + let long = format!("a/{}", "b".repeat(125)); + assert_eq!(mime_problem(&long), None); + for bad in [ + "pdf", + "/pdf", + "a/", + "a/b\r\nX: y", + "a/é", + &format!("{long}b"), + ] { + assert!(mime_problem(bad).is_some(), "{bad:?}"); + } + } + + #[test] + fn bytes_changed_after_they_came_are_never_filed() { + let dir = tempfile::tempdir().unwrap(); + let u = Uploads::open(dir.path(), NOW).unwrap(); + let bytes = b"%PDF-1.4 as sent".to_vec(); + let g = grant_for(&u, "tok_a", &bytes); + u.receive(&g.id, "tok_a", &mut &bytes[..], NOW).unwrap(); + let bin = dir.path().join(UPLOADS_DIR).join(format!("{}.bin", g.id)); + std::fs::write(&bin, b"%PDF-1.4 as kept").unwrap(); + assert_eq!( + u.take(&g.id, "tok_a", NOW).unwrap_err(), + UploadError::WrongHash + ); + assert!(matches!( + u.take(&g.id, "tok_a", NOW).unwrap_err(), + UploadError::Unknown(_) + )); + } + + #[test] + fn making_a_grant_sweeps_the_lapsed() { + let dir = tempfile::tempdir().unwrap(); + let u = Uploads::open(dir.path(), NOW).unwrap(); + let old = grant_for(&u, "tok_a", b"old"); + u.receive(&old.id, "tok_a", &mut &b"old"[..], NOW).unwrap(); + u.grant( + "tok_a", + &sha256_hex(b"new"), + 3, + "text/plain", + "x", + "receipt", + NOW + UPLOAD_TTL, + ) + .unwrap(); + let names: Vec = std::fs::read_dir(dir.path().join(UPLOADS_DIR)) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().to_string()) + .collect(); + assert!(names.iter().all(|n| !n.starts_with(&old.id)), "{names:?}"); + assert_eq!(names.len(), 1); + } + + #[test] + fn a_token_has_only_so_many_uploads_under_way() { + let dir = tempfile::tempdir().unwrap(); + let u = Uploads::open(dir.path(), NOW).unwrap(); + let sha = sha256_hex(b"x"); + for _ in 0..MAX_LIVE { + u.grant("tok_a", &sha, 1, "text/plain", "x", "w9", NOW) + .unwrap(); + } + let e = u + .grant("tok_a", &sha, 1, "text/plain", "x", "receipt", NOW) + .unwrap_err(); + assert!(matches!(e, UploadError::Bad(_)), "{e}"); + // Another token's are its own; lapsed ones don't count. + u.grant("tok_b", &sha, 1, "text/plain", "x", "receipt", NOW) + .unwrap(); + u.grant( + "tok_a", + &sha, + 1, + "text/plain", + "x", + "receipt", + NOW + UPLOAD_TTL, + ) + .unwrap(); + } + + #[cfg(unix)] + #[test] + fn its_folder_and_files_are_private() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let u = Uploads::open(dir.path(), NOW).unwrap(); + let g = grant_for(&u, "t", b"abc"); + let mode = |p: &Path| std::fs::metadata(p).unwrap().permissions().mode() & 0o777; + assert_eq!(mode(&dir.path().join(UPLOADS_DIR)), 0o700); + assert_eq!( + mode(&dir.path().join(UPLOADS_DIR).join(format!("{}.json", g.id))), + 0o600 + ); + // The bytes, kept and held, are private too. + let folder = dir.path().join(UPLOADS_DIR); + u.receive(&g.id, "t", &mut &b"abc"[..], NOW).unwrap(); + assert_eq!(mode(&folder.join(format!("{}.bin", g.id))), 0o600); + let held = u.take(&g.id, "t", NOW).unwrap(); + assert_eq!(mode(&folder.join(format!("{}.{TAKEN}", g.id))), 0o600); + drop(held); + } +} diff --git a/docs/mcp-interface.md b/docs/mcp-interface.md index 23abcb8..5e137ab 100644 --- a/docs/mcp-interface.md +++ b/docs/mcp-interface.md @@ -191,6 +191,57 @@ settings-and-conventions section says how to use them. binding a LAN, tailnet or public interface is an explicit human decision, never a default. Auth, scopes and Origin validation arrive with the served books (`plan/served.md`, P2). +- **Served books** (`wanderbooks server`, started by the host; an agent + connects with `wanderbooks mcp --book ID`, a stdio bridge): every book + in the data root, on a socket in the root's private `run/` folder that + only this account can connect to. Every request carries a bearer token + live for that book (refused 401 without one, 403 with another book's). + Refusals at the door, tool calls, uploads and downloads are recorded + in the book's access log; when a record can't be written the book + stops taking requests (an upload or download is then refused, 503). + - `/b/{book}/mcp`: the book's MCP service. A request is at most 1 MiB. + - `PUT /b/{book}/u/{upload}`: an upload's bytes (`books.begin_upload`), + from the token that began it; what `wanderbooks upload --book B + [--token NAME] UPLOAD FILE` sends. Streamed to the disk, kept only if + exactly the size and SHA-256 declared (so at most 25 MiB), once. The + receive and its access record are one blocking task: bytes are never + kept without their record (if it can't be written they are thrown + away, 503 `not-recorded`, and the books stop). 200 with `{ received, + size, sha256 }`; else, with nothing kept and the code in + `x-wanderbooks-refused`: 404 `no_such_upload` (unknown or lapsed, + also if it lapsed while being sent), 403 `not_yours`, 409 + `already_sent` (its bytes came, or a call holds them), 409 `in_use` + (another request is sending them), 400 `not_as_declared` (short, more than declared, or + the wrong hash), 503 `stopped` (the books stopped after an internal + error), 500 `failed`. A refused request's body is read and dropped + (up to 25 MiB), so the client gets the answer, not a cut connection. + Logged as action `upload` with the upload's id, a refusal at the + door (no token, another book's) too. An upload's id is `up_` and 32 + letters `a`-`p`: no digits, so the access log's TIN masking never + hides it. + - `GET /b/{book}/a/{attachment}`: a stored file's bytes, for a token + with `books:read` (403 `insufficient_scope` without it; 404 + `no_such_attachment`); the read and its access record are one + blocking task. Sent with `Content-Type` (the stored MIME type, or + `application/octet-stream` if it can't be a header), + `Content-Disposition: attachment; filename="…"; filename*=UTF-8''…` + (the plain name: anything but visible ASCII and spaces, and `"` and + `\`, made `_`; then the whole name, RFC 8187 percent-encoded), + `X-Content-Type-Options: nosniff`, and `Repr-Digest: + sha-256=::` (RFC 9530). A stored file that can't be read + whole is 500 `failed`; books that stopped, 503 `stopped`. GET only + (a HEAD is 405, unrecorded). Logged as + action `download` with the attachment's id, a refusal at the door + too. + `wanderbooks download --book B [--token NAME] ATTACHMENT --to DIR` + asks for it, checks the digest before saving, and saves it under the + server's name (`filename*` first) made plain: separators and control + characters made `_`, invisible (Unicode format) characters and + leading dots and spaces dropped, at most 200 bytes with the extension + kept. Beside a file of that name it is `name (1).ext`, and so on; + `attachment-N` if the folder takes no such name. It is written to a + hidden file and hard-linked in: never a partial file under the real + name, never over another file. ## Server instructions @@ -256,7 +307,7 @@ No text the agent is given (tool JSON with its schemas, resources, connect text, handbook, and every refusal fixed in code: each error variant's template in the crates under the server, scanned from source, built error texts, and the server's refusal constants) names a helper command, a home -path, the data dir or its flag, the import-root flag, a file of the books +path, the data dir or its flag, the old import-root flag, a file of the books (`.jsonl`, the database, `vault.key`, `.gitignore`), git, GitHub, a pull request, a repo doc (`docs/`, `.md`), or tells the agent to read the books' files; no tool title or description, connect text or handbook @@ -614,14 +665,17 @@ texts are at most 500 characters, no control characters. `party.role_set`. Removing the last 1099 role with W-9 facts on file, or employee with W-4 facts, is refused: clear the facts first. - `books.set_w9 { party, received_on, entity_type, exempt_payee_code?, - fatca_code?, tin_last4, kept_at, tin?, tin_type?, w9_file_path? }` or - `{ party, clear: true }` → the party. Journaled as `party.w9_set`. A - party with a vendor record also gets the record's `tin_on_file` set to - match (journaled as `vendor.1099_set`), so `books.vendor_1099`'s - `missing_w9` follows. `tin`, `tin_type`, and `w9_file_path` put the full - TIN and the signed form in the TIN vault (below); with them `tin_last4` - comes from the TIN (a given one must agree) and `kept_at` defaults to - `"wanderbooks vault"`. + fatca_code?, tin_last4, kept_at, tin?, tin_type?, w9_upload_id?, + w9_content_base64?, w9_sha256?, w9_file_name? }` or `{ party, clear: + true }` → the party. Journaled as `party.w9_set`. A party with a + vendor record also gets the record's `tin_on_file` set to match + (journaled as `vendor.1099_set`), so `books.vendor_1099`'s + `missing_w9` follows. + `tin` and `tin_type` put the full TIN in the TIN vault, and the signed + form (`w9_upload_id`, or `w9_content_base64` with `w9_sha256`, as in + "Tools — receipts & documents") goes there too (below); with them + `tin_last4` comes from the TIN (a given one must agree) and `kept_at` + defaults to `"wanderbooks vault"`. - `books.set_w4 { party, received_on, state, kept_at }` or `{ party, clear: true }` → the party. Journaled as `party.w4_set`. - `books.set_entry_party { entry_id, party?, parties?, no_party_reason? }` @@ -749,56 +803,58 @@ and the CPA package carries them. facts name another last four is refused (a payee's new TIN comes with its W-9: `books.set_w9`); one whose W-9 facts agree gets their `vault_record` updated too (`party.w9_set`). -- `books.set_w9 { ..., tin, tin_type, w9_file_path? }` (above): validates - the TIN against its type (nine digits, hyphens allowed; an ITIN starts - with 9 and has an IRS group in digits 4-5; an SSN or EIN doesn't start - with 9), checks the facts on a copy of the ledger, then writes the vault - and journals the facts with `vault_record`, the SHA-256 of the TIN - record (of the form's record when only a form is given). With a TIN it - also journals the party's TIN facts (`party.tin_set`, as - `books.set_tin`); a TIN with a new last four clears the old TIN facts - first (`party.tin_set` null), so the two never disagree. The form (PDF, - PNG, JPEG; 25 MB at most) goes to the vault only, never the attachment - store. `w9_file_path` is read exactly as `books.import_local_file` reads - a path (the import roots, the sealed places, no links followed out of - them), and its type is judged by the file the path really leads to. - Facts given later without a TIN keep the vault record when their - last four agree with the vault's TIN, and are refused when they don't - (pass the new TIN, or purge). Clearing the facts (`clear: true`) is - refused while the vault holds the party's TIN or form: purge first. +- `books.set_w9 { ..., tin, tin_type, w9_upload_id? | w9_content_base64? + + w9_sha256?, w9_file_name? }` (above): validates the TIN against its type (nine digits, + hyphens allowed; an ITIN starts with 9 and has an IRS group in digits + 4-5; an SSN or EIN doesn't start with 9), checks the facts on a copy of + the ledger, then writes the vault and journals the facts with + `vault_record`, the SHA-256 of the TIN record (of the form's record when + only a form is given). With a TIN it also journals the party's TIN facts + (`party.tin_set`, as `books.set_tin`); a TIN with a new last four clears + the old TIN facts first (`party.tin_set` null), so the two never + disagree. The form (a PDF, PNG or JPEG, judged by its own first bytes, + whatever it is called; an upload of purpose `w9`, at most 25 MiB, or + inline at most 256 KiB) goes to the vault only, never the attachment + store. Its file name is `w9_file_name` if given, else the upload's, + else `w9.pdf`, `w9.png` or `w9.jpg` by its type, checked as any filed + name; `w9_file_name` without a W-9 is refused. Facts given later without a TIN + keep the vault record when their last four agree with the vault's TIN, + and are refused when they don't (pass the new TIN, or purge). Clearing + the facts (`clear: true`) is refused while the vault holds the party's + TIN or form: purge first. - `books.get_w9 { party, save_to }` → `{ party_id, path, file_name, - size_bytes, note }`. Writes the party's form from the vault to `save_to`: - a new file (an existing one is refused), in a folder that exists, - outside the books dir and outside the import roots, mode 0600. Returns - the path, never the contents. Journals nothing. The same holds for - `books.vendor_1099`'s `save_csv_to` with `include_tins` and - `books.export_cpa_package`'s `save_to` with TINs: a file with full TINs - never lands where it could be filed back in. Nor does one go in from - elsewhere: the server keeps, beside the vault, the SHA-256 of every file - it read or wrote with full TINs (the W-9 `books.set_w9` reads, recorded - once the call has passed every check, just before the vault keeps it, - so a refused call marks no file; each of these outputs; each file inside a CPA package - that holds a TIN), shared by every session and kept across restarts, - and every filing tool (`books.import_local_file`, `books.attach_receipt`, - `books.file_document`) refuses those exact files and any text file - (CSV, TSV, text, JSON, IIF, XML, HTML, Markdown, by type, name or UTF-8 - or UTF-16 decoding, UTF-16 with a byte-order mark or NUL-interleaved - without one) holding a full TIN, storing nothing. A text file - holds one when it has a TIN the vault holds, anywhere, as nine digits, - 3-2-4 joined by hyphens, spaces or dots, or 2-7 joined by a hyphen or a - space (compared on the server; a vault that can't be opened leaves the - next test alone), or a stand-alone number printed as an SSN or EIN is + size_bytes, note }`. Writes the party's form from the vault to + `save_to`: a new file (an existing one is refused), in a folder that + exists, outside the books dir (where it really leads, every link + followed), mode 0600. Returns the path, never the contents. Journals + nothing. The same holds for `books.vendor_1099`'s `save_csv_to` with + `include_tins` and `books.export_cpa_package`'s `save_to` with TINs: a + file with full TINs never lands inside the books. Nor does one go in + from elsewhere: the server keeps, beside the vault, the SHA-256 of every + file it read or wrote with full TINs (the W-9 `books.set_w9` reads, + recorded once the call has passed every check, just before the vault + keeps it, so a refused call marks no file; each of these outputs; each + file inside a CPA package that holds a TIN), shared by every session and + kept across restarts, and every filing tool (`books.attach_receipt`, + `books.file_document`) refuses those exact files and any text file (CSV, + TSV, text, JSON, IIF, XML, HTML, Markdown, by type, name or UTF-8 or + UTF-16 decoding, UTF-16 with a byte-order mark or NUL-interleaved + without one) holding a full TIN, storing nothing. A text file holds one + when it has a TIN the vault holds, anywhere, as nine digits, 3-2-4 + joined by hyphens, spaces or dots, or 2-7 joined by a hyphen or a space + (compared on the server; a vault that can't be opened leaves the next + test alone), or a stand-alone number printed as an SSN or EIN is (123-45-6789, 12-3456789; no digit, and no hyphen or dot followed by a - digit, beside it). Bank exports pass: routing and account numbers, - long references and dates next to amounts are not TINs (rules.md B-26). - No probing: a text with more than two TIN-shaped numbers ending in the - last four of a TIN the books know is refused before the vault is - opened, in the same words whether or not one is that TIN, so the vault - is compared against at most two candidates a filing (rules.md B-26). - An altered copy of a binary file (a W-9 PDF re-saved or re-scanned), - and a TIN the vault doesn't hold written another way, are not - detected: P2 and P7 take the path tools and full TINs out of the - agent's reach. + digit, beside it). Bank exports pass: routing and account numbers, long + references and dates next to amounts are not TINs (rules.md B-26). No + probing: a text with more than two TIN-shaped numbers ending in the last + four of a TIN the books know is refused before the vault is opened, in + the same words whether or not one is that TIN, so the vault is compared + against at most two candidates a filing (rules.md B-26). An altered copy + of a binary file (a W-9 PDF re-saved or re-scanned), and a TIN the vault + doesn't hold written another way, are not detected: P2 slice 6 moves the + TIN outputs (`save_to`, `save_csv_to`) to the viewer, and P7 takes full + TINs out of the agent's reach. - `books.purge_tin { party, reason, confirmed? }` → `{ status: "preview" | "purged", party_id, name, tin_in_vault, w9_file_in_vault, tin_last4?, journaled, note }`. Without `confirmed`, a preview that changes nothing. @@ -841,10 +897,56 @@ the receipt policy counts it. ## Tools — receipts & documents -- `books.attach_receipt { entry_id?, file_name, mime_type, content_base64, - description?, link_to? }` → `{ attachment_id, sha256, deduplicated, links - }`. `entry_id` is optional: a receipt can be filed before its entry - exists. `link_to` as in "Tools — attachments". +A file comes in one of two ways, named in the tool that files it +(`books.attach_receipt`, `books.file_document`, `books.set_w9`, +`books.import_bank_csv`, `books.import_chart_csv`): `upload_id`, an +upload made with `books.begin_upload` (any file up to 25 MiB; its bytes +never pass through the agent), or `content_base64` with its `sha256` +(at most 256 KiB decoded; longer text is refused before it is decoded). +Never both, never neither. Inline bytes that aren't base64, or whose +SHA-256 isn't the one given, are refused (guard `base64_corrupt`): send +the file as an upload. Uploads exist only for books served from the +data root (the server's socket carries the bytes); over stdio, only +inline, and the refusal says so and to tell the human. On the socket an +MCP request is at most 1 MiB. A file's name and type are checked when +it is filed (every filing tool, `books.begin_upload`, and the W-9's +name): a name of 1 to 255 bytes, with no `/` or `\`, not `.` or `..`, +and no control or invisible (Unicode format) characters; a MIME type +`type/subtype` (parameters allowed) of visible ASCII, at most 127 bytes. + +- `books.begin_upload { sha256, size, mime_type, file_name, purpose: + "receipt" | "document" | "import" | "w9" }` → `{ upload_id?, command?, + expires_at?, max_bytes, existing_attachment_id? }`. Makes a grant for + the calling token: one SHA-256, one size (at most 25 MiB; empty + refused), one purpose, lasting 15 minutes; at most 16 under way per + token. A tool takes only an upload of its own purpose: + `books.attach_receipt` and `books.file_document` take `receipt` or + `document`, the CSV imports `import`, `books.set_w9` `w9`; another + tool refuses it and leaves it for the right one. `command` is + `wanderbooks upload --book B -- ` (with + `--token NAME` when the caller's token isn't the default `agent`), + which sends the bytes (`PUT /b/{book}/u/{upload}`, "Transports"); the + agent runs it, then names `upload_id` in the tool that files it. + Bytes that aren't the size and SHA-256 declared are refused and + nothing is kept. An upload's bytes are sent once, only by the token + that began it, and used once: the call that takes them holds them + (hashed again) and spends them only when they are in the books (an + import: once its rehearsal passed); a call refused after taking them, + or one that panics, gives them back, and a server stopped mid-call + gives them back when the books next open. Lapsed grants are refused, + and swept when the books open, whenever a grant is made, and every + 10 seconds. For `receipt` or `document`, when an attachment not withdrawn + already holds those bytes, no grant is made: `existing_attachment_id` + names it (link it, `books.link_attachment`, instead of filing a copy); + for `import` and `w9` a grant is always made. Refusals never name a + path. +- `books.attach_receipt { entry_id?, file_name?, mime_type?, upload_id? | + content_base64? + sha256?, description?, link_to? }` → `{ attachment_id, + sha256, deduplicated, links }`. `entry_id` is optional: a receipt can be + filed before its entry exists. `file_name` and `mime_type` default to + the upload's; inline they are required. The upload is taken only after + the entry and every `link_to` subject check out. `link_to` as in "Tools + — attachments". - `books.link_receipt { receipt_id, entry_id }` → link a previously filed (unlinked, not withdrawn) receipt to its entry. A receipt linked elsewhere must be unlinked first; one already linked to that entry @@ -865,15 +967,23 @@ the receipt policy counts it. subject it is linked to, its filing entry first). Withdrawn attachments are omitted unless `include_withdrawn` is true. `entry_id` lists the files filed on the entry and linked to it. -- `books.get_receipt { attachment_id }` → `{ metadata, description?, - links, content_base64 }`. +- `books.get_receipt { attachment_id, view?: "meta" | "bytes" }` → `{ + metadata, description?, links, content_base64?, download? }`. `meta` + (the default) returns no bytes. `bytes` adds `content_base64` + (hash-verified) for a file of at most 256 KiB; a larger one gives + `download`, the command `wanderbooks download --book B --to ` (with `--token NAME` when the caller's token isn't the + default `agent`; `GET /b/{book}/a/{attachment}`, "Transports"), which + saves a copy as a new file in that folder. Over stdio there is no + download: a larger file is refused. - `books.file_document { category: "formation" | "legal" | "tax" | "accounting" | "statement" | "contract" | "photo" | "warranty" | - "import_source" | "other", file_name, mime_type, content_base64, - description?, link_to? }` → `{ attachment_id, sha256, deduplicated, links - }`. The Drive-folder replacement (today's 01 Formation / 02 Legal / 03 - Tax / 04 Accounting). Never a W-9: it shows the full TIN (`books.set_w9 - w9_file_path`). + "import_source" | "other", file_name?, mime_type?, upload_id? | + content_base64? + sha256?, description?, link_to? }` → `{ attachment_id, + sha256, deduplicated, links }`. The bytes, name and type as in + `books.attach_receipt`. The Drive-folder replacement (today's 01 + Formation / 02 Legal / 03 Tax / 04 Accounting). Never a W-9: it shows + the full TIN (`books.set_w9 w9_upload_id`). ## Receipt enforcement @@ -977,51 +1087,19 @@ posting. ## Tools — import -- `books.import_bank_csv { csv_text, bank_account_code, default_offset_code, - rules?, date_order?, source_attachment_id? }` → creates one draft per - transaction (QBO upload layout). `rules` maps description substrings to - account codes. `source_attachment_id` (the export as filed, category +- `books.import_bank_csv { csv_text? | upload_id?, bank_account_code, + default_offset_code, rules?, date_order?, source_attachment_id? }` → + creates one draft per transaction (QBO upload layout). `rules` maps + description substrings to account codes. `source_attachment_id` (the export as filed, category `import_source`) is checked before any draft and linked to the bank account (`source_linked`; false when it was already), never to the drafts: a bank export is not a receipt. -- `books.import_chart_csv { csv_text }` → creates accounts from - `code,name,type,parent_code` CSV. All-or-nothing. -- `books.import_local_file { path, category: "receipt" | "document", ... }` - → files a file from the server's own disk, read only from the **import - roots**: each `--import-root DIR` given to `ledger-mcp` (repeatable), or - by default the downloads folder only (`$XDG_DOWNLOAD_DIR`, else - `~/Downloads`). No root may be `/`, the home folder or a folder holding - it, a folder holding the books dir, the data dir (where a symlinked one - really is), the vault or a vault key, or a folder inside the books dir, - the data dir or the vault (every file there would be refused): a named - root like that stops - the server at startup, saying why; a default one is dropped. The default - is used only when it is a real folder that isn't the home folder (an - `XDG_DOWNLOAD_DIR` of `$HOME/` falls back to `~/Downloads`, and to no - root when that doesn't exist; an import then says the server has no - import folder). The path must be absolute, with no `.` or `..` name, - and where it leads (every link followed) a regular file strictly inside - a root, with one hard link. Refusals never name a path and never probe: - a path that is missing, a dangling link, outside the roots, or with a - `.` or `..` name (refused before anything is looked up: `a/b/..` - resolves only when `b` is a folder) gets one refusal, "no - such file in the server's import folders (by default your downloads - folder): save it there first". Inside a root, the books' sealed places - (the data dir, the vault, the key file in force and the default one, - where each leads, `~/.config/wanderbooks`, and the key files' folders - unless that folder is the home folder, a root, or holds one) are refused - as "belongs to the books or the TIN vault"; anything else in the books - dir is refused with "save a copy of it in the server's import folders - first". The file is opened by a walk from `/`, one name at a time, each - folder with `openat(O_DIRECTORY | O_NOFOLLOW)` and the file with - `O_NOFOLLOW | O_NONBLOCK`, so a folder on the path swapped for a link - after the check fails the open and a pipe can't hang the server; no step - may land on a protected place by device and inode (one renamed into a - root), and the file opened must be the checked one (device, inode, one - link). It is filed under the name and type of the file the path leads - to. Files holding full TINs are refused as above - (`ledger-mcp/src/local_files.rs`, `tin_files.rs`). Deleted in P2 - (uploads replace it). +- `books.import_chart_csv { csv_text? | upload_id? }` → creates accounts + from `code,name,type,parent_code` CSV. All-or-nothing. +- Each takes the CSV as `csv_text` or as an upload (`books.begin_upload` + purpose `import`; UTF-8), never both. The CSV itself is not filed: file + it with `books.file_document` (category `import_source`) for + `source_attachment_id`. ## Resources (read-only) @@ -1036,7 +1114,8 @@ Hand-rolled on `ServerHandler` (rmcp 3.x has no resource macros): this month so far - `books://handbook` (the index) and `books://handbook/{section}` (one per section, Markdown) -- `books://receipts/{id}` — receipt metadata + base64 bytes +- `books://receipts/{id}` — receipt metadata (as `books.get_receipt` + view `meta`; no bytes) Resources exist so read-only clients (dashboards, the future read-only GUI, auditors) never need tool credentials. Tested end-to-end over stdio. diff --git a/docs/parity.md b/docs/parity.md index 0964060..9df185e 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -57,8 +57,9 @@ version refuses books from a newer one as newer, not corrupt (rules.md B-27); the MCP server fails closed after an internal error, and no input reaches one (dates are four-digit years, amounts at most a line's, B-28); local files -are read only from the import roots, never the books, the vault or its -key; no file with a full TIN is filed into the books (B-26); the agent is +were read only from the import roots, never the books, the vault or its +key (P2 slice 5 removed local paths: files now come in as uploads or +inline bytes); no file with a full TIN is filed into the books (B-26); the agent is told the rules in force in a short connect text and a handbook, and no agent text points at the books' files, git or the repo. diff --git a/docs/parties-and-attachments.md b/docs/parties-and-attachments.md index 3f1e97a..230ffcb 100644 --- a/docs/parties-and-attachments.md +++ b/docs/parties-and-attachments.md @@ -275,8 +275,9 @@ damn CPA package." open and verify; the 1099 worksheet and the CPA package then say which TINs are missing (`tin_gaps`), and the package is still built. - **Tools** (built; `docs/mcp-interface.md` "Tools — the TIN vault"): - `books.set_w9` takes `tin` + `tin_type` and `w9_file_path` (the form - goes to the vault, never the attachment store) and journals the last + `books.set_w9` takes `tin` + `tin_type` and the signed form as + `w9_upload_id` (`books.begin_upload`) or `w9_content_base64` with + `w9_sha256` (the form goes to the vault, never the attachment store) and journals the last four with `vault_record`; facts that disagree with the vault's TIN are refused, and clearing facts with a vault record needs a purge first. With a TIN it also records the party's TIN facts. `books.set_tin diff --git a/docs/receipt-store.md b/docs/receipt-store.md index bbb6946..c7a0695 100644 --- a/docs/receipt-store.md +++ b/docs/receipt-store.md @@ -89,8 +89,13 @@ before backups. - **Deduplicated.** Attaching the same receipt twice stores one blob and two attachment rows. `attach_receipt` / `file_document` return `deduplicated: true` when the blob already existed. -- **Limits.** 25 MB per file by default, configurable. MIME type is accepted - from the client and stored; server-side sniffing is future work. +- **Limits.** 25 MiB per file as an upload (`books.begin_upload`), 256 + KiB given inline. The name and MIME type are the client's, checked + when the file is filed: a name of 1 to 255 bytes, no `/` or `\`, not + `.` or `..`, no control or invisible characters; a `type/subtype` of + visible ASCII, at most 127 bytes (`docs/mcp-interface.md`, "Tools — + receipts & documents"). The type is stored as given; server-side + sniffing is future work (only a W-9 is judged by its first bytes). - **Metadata** lives in SQLite (`blobs` table), not in the filesystem: `sha256 PK, size_bytes, mime_type, original_name, created_at`. diff --git a/docs/spec/rules.md b/docs/spec/rules.md index 9e95088..1ff6d9c 100644 --- a/docs/spec/rules.md +++ b/docs/spec/rules.md @@ -583,10 +583,11 @@ asked): a refusal that quotes its input masks any TIN in it and a party text field refuses nine digits however they are separated (space, hyphen, dot, slash, underscore; `has_full_tin`). Files holding one (`books.get_w9`, `save_csv_to`, and the CPA package, whose K-1 and 1099 worksheets carry every member's and payee's -TIN by default) are written only outside the books dir and outside the -server's import folders (DECISION, P0 review, 2026-10-16: so none can be -filed back in as a receipt and read out again), and the package comes -back only as its path. No file holding a full TIN is filed into the books +TIN by default) are written only outside the books dir (DECISION, P0 +review, 2026-10-16: so none can be filed back in as a receipt and read +out again; the server's import folders, refused then too, went with +`books.import_local_file` in P2 slice 5), and the package comes back +only as its path. No file holding a full TIN is filed into the books (DECISION, P0 review round 2, 2026-10-16): the server keeps, beside the vault, the SHA-256 of every file it read or wrote with full TINs (the W-9 `books.set_w9` reads, recorded once the call has passed every check, just @@ -594,8 +595,8 @@ before the vault keeps it, so a refused call marks no file; each `books.get_w9` copy; a 1099 worksheet with TINs; a CPA package with TINs and each file in it that holds one), shared by every session and every later server for the -books, and every way a file enters the books (`books.import_local_file`, -`books.attach_receipt`, `books.file_document`) refuses those exact files, +books, and every way a file enters the books (`books.attach_receipt`, +`books.file_document`, an upload or inline) refuses those exact files, and refuses any text file (by its type or name, or because it decodes as UTF-8 or UTF-16, with a byte-order mark or, NUL-interleaved, without one) holding (a) a TIN the vault holds for a party of the diff --git a/docs/testing.md b/docs/testing.md index 5b5e357..f035ad5 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -65,13 +65,10 @@ that would fail if it broke: | No input reaches a panic (each would stop the server for every session): every tool with arguments generated from its schema and made hostile (wrong types, u64::MAX, decimals of 30+ digits or absurd exponents, dates like 0000-00-00, 99999-12-31, +262142-12-31, 2026-02-30, periods like 999999-01, control and format characters, long strings), every resource URI generated; the inputs that once panicked, pinned; after each property the server restarts and the books reopen (a call a command accepts but replay refuses would brick them); every tool and resource over every old book (rules.md B-28) | `ledger-mcp` `tests/hostile_args.rs` `hostile_tool_arguments_never_stop_the_server` (with a floor: most calls reach a tool, some succeed), `hostile_resource_uris_never_stop_the_server`, `inputs_that_once_panicked_are_refused_or_answered`, `old_books_answer_every_tool`; `MCP_CASES` and `OLD_BOOKS_CASES` (default 3 per tool per book) set the cases | | Dates are in 0001-01-01 through 9999-12-31 and amounts at most a line's, for every new command; replay accepts every line an earlier version accepted (each bound has its pair: the command refuses past it, the replayed old line is accepted); a figure derived past exact arithmetic or chrono's calendar is refused, never a panic; a recurring run makes at most 1,000 drafts (rules.md B-28) | `ledger-core` `tests/spec_bounds.rs`, `calendar.rs` tests, `plans_at_the_edges_are_computed_or_refused`, `intervals_reach_chronos_last_day`, `a_vendor_part_is_scaled_exactly_or_refused`, the `share_property` cases; `ledger-store` `tests/bounds.rs`, `tests/store.rs` (the old-books tests); `server.rs` `periods_are_a_year_and_a_month`, `dates_are_four_digit_years_and_real_days` | | Opening the books changes nothing until every complete line passes in memory (chain, gate, first line, replay); the viewer applies the writer's first-line rule (rules.md B-27) | `format.rs` `a_bad_complete_line_before_a_torn_tail_leaves_the_journal_as_it_was`, `the_first_line_is_store_created_for_the_viewer_and_the_writer`; `fuzz_store_replay` (the first line varies) | -| A path with a `.` or `..` name is refused as missing before anything is looked up; a root inside the books, the data dir or the vault is refused at startup (dropped as the default) | `local_files.rs` `a_dot_or_dotdot_name_never_tells_whether_folders_outside_exist`, `a_root_holding_the_books_or_the_home_folder_is_refused`; `startup.rs` `an_import_root_holding_the_books_or_the_home_folder_is_refused`, `a_default_download_dir_holding_the_books_is_dropped_not_fatal` | | UTF-16 text without a byte-order mark is scanned like any text; a refused `books.set_w9` marks no file; a refused resource read never echoes a TIN | `tin_files.rs` (tests) `utf16_text_without_a_byte_order_mark_is_scanned_either_way_round`, `a_refused_set_w9_never_marks_its_file`, unit test `only_nul_interleaved_bytes_read_as_utf16_without_a_mark`; `spec_vault.rs` `resource_refusals_never_echo_a_tin` | | idempotentHint only where a repeat with the same arguments leaves the journal and the vault as they were; every tool's hints pinned in one table | `spec_vault.rs` `idempotent_tools_change_nothing_when_repeated`; `annotations_complete` | | books.settings lists every receipt-required and exempt account, uncut; a fresh company is "a new company"; the lint catches `just` however it ends and where the books are | `mcp_stdio.rs` `every_receipt_account_left_out_of_the_connect_text_is_in_books_settings`; `handbook.rs` `a_fresh_company_is_a_new_company`; `tool_text_lint_catches_helpers_and_locations` | -| A local file is read only from inside an import root, never a sealed place (data dir, vault, keys, the key folders that don't hold a root), never through a swap after the check (a component walk with `O_NOFOLLOW`, no step onto a protected inode); a missing, dangling or outside path gets one refusal | `local_files.rs` `import_local_file_refuses_key_and_journal`, the property `import_allowed_iff_regular_file_inside_a_root_outside_the_books`, `refusals_never_tell_what_is_there`, `a_file_swapped_after_the_check_is_never_read`, `a_folder_swapped_after_the_check_is_never_walked_through` (the reviewer's race, without `/proc`), `resolved_follows_every_link_a_dangling_one_too`, `sealing_a_key_never_seals_a_whole_root`; through the tools, `mcp_stdio.rs` `import_local_file_and_set_w9_refuse_the_key_vault_and_journal_inside_a_root` | -| No import root is `/`, the home folder or holds it, or holds the books, data dir, vault or a key; the default is a real downloads folder or none | `local_files.rs` `the_default_root_is_never_the_home_folder_or_the_filesystem`, `a_root_holding_the_books_or_the_home_folder_is_refused`; `startup.rs` `an_import_root_holding_the_books_or_the_home_folder_is_refused`, `a_download_dir_of_home_never_opens_the_home_folder` | -| A file with full TINs is never written into an import root, and never filed into the books: every file the server read or wrote with TINs (from any session, after a restart, out of a CPA package), by every filing tool | `mcp_stdio.rs` `tin_bearing_files_never_land_where_they_could_be_filed_back_in`; `tin_files.rs` (tests) `a_tin_bearing_file_is_refused_from_every_session_and_after_a_restart`; unit test `a_recorded_tin_file_is_refused_by_every_reader_of_the_registry` | +| A file with full TINs is never written inside the books, and never filed into the books: every file the server read or wrote with TINs (from any session, after a restart, out of a CPA package), by every filing tool | `mcp_stdio.rs` `tin_bearing_files_never_land_where_they_could_be_filed_back_in`; `tin_files.rs` (tests) `a_tin_bearing_file_is_refused_from_every_session_and_after_a_restart`; unit test `a_recorded_tin_file_is_refused_by_every_reader_of_the_registry` | | Text filed into the books is refused when it holds a TIN the vault keeps (nine digits, 3-2-4 by hyphen, space or dot, 2-7 by hyphen or space, anywhere) or a stand-alone printed SSN or EIN (123-45-6789, 12-3456789), and only then: a bank export (routing numbers, long references, a date next to an amount) and a longer number holding an SSN shape go in; a locked vault leaves the printed shapes (rules.md B-26) | `tin_files.rs` (tests) `text_with_a_vault_tin_or_a_printed_one_is_refused_and_a_bank_export_is_not`; unit tests `text_holding_a_vault_tin_is_refused_in_every_rendering`, `a_printed_ssn_or_ein_is_refused`, `a_longer_number_holding_a_tin_shape_passes`, `an_ordinary_bank_export_passes`, `binary_bytes_are_not_scanned` | | Filing is no TIN oracle: text with more than two TIN-shaped numbers ending in a known TIN's last four (the books' TIN and W-9 facts, then the vault's own) is refused before the vault is opened, byte for byte alike whether one is the vault's TIN or none is, the vault open or locked, by every filing tool; one or two such numbers are compared against the vault, one number written many ways counts once, and other last fours never count (rules.md B-26) | `tin_files.rs` (tests) `text_probing_for_a_tin_is_refused_alike_by_every_filing_tool`; unit tests `text_probing_for_a_tin_is_refused_alike_whether_or_not_it_holds_it` (the vault never opened for a probe), `one_or_two_candidates_are_compared_against_the_vault` | | The connect text fits its budgets, its lead stands alone, it states the receipt rule enforced (exact threshold in the base currency, required and exempt accounts, reversals) and the parties rule in force, and the company name can't break it (control and format characters) | `handbook.rs` `instructions_budget`, `a_typical_company_gets_the_whole_connect_text`, `the_company_name_is_one_plain_line`; `receipt_policy.rs` `the_rule_told_is_the_rule_enforced`, `a_long_or_odd_account_list_stays_short_and_plain`; a new session is told the settings in force: `a_new_session_is_told_the_settings_in_force` | @@ -110,19 +107,17 @@ broke, removed by simpler code, or is equivalent: - **Killed (MCP).** A section's title is its heading (`a_title_is_the_heading_text`; over stdio the handbook tool titles a section by its heading and "all" as the whole handbook, - `handbook_tool_and_resources_over_stdio`). A refused default import - root is dropped, never fatal (`ImportRoots::explicit`: - `a_default_download_dir_holding_the_books_is_dropped_not_fatal`). A - `.jpg` or `.jpeg` W-9 is taken (`every_w9_file_type_is_taken`). The - waker wakes: a request pending until another thread wakes it completes - (`a_request_that_waits_is_woken_and_completes`). The walk opens each - folder with `O_DIRECTORY | O_NOFOLLOW`: a folder swapped for a link to - an unprotected folder outside the roots, or for a pipe, is never walked - through and never blocks - (`a_folder_swapped_for_a_link_or_a_pipe_is_never_walked_through`). A - read failure names its kind, never the path - (`cant_read_names_the_failure_not_the_path`). `resolved` follows 40 - links, as the kernel does, and no more + `handbook_tool_and_resources_over_stdio`). A `.jpg` or `.jpeg` W-9 + was taken (`every_w9_file_type_is_taken`); P2 slice 5 judges the W-9 + by its first bytes, not its name, so that mutant's code is gone (the + test now checks each type by its bytes). The waker wakes: a request pending until another thread wakes it completes + (`a_request_that_waits_is_woken_and_completes`). Three more were + killed by tests of the import roots and their walk (a refused default + root dropped, never fatal; a folder swapped for a link or a pipe never + walked through; a read failure naming its kind, never the path): P2 + slice 5 removed `books.import_local_file` and the import roots, and + those tests with them. `resolved` follows 40 links, as the kernel + does, and no more (`resolved_follows_forty_links_and_no_more`). A file is saved only to a new file in a folder that exists, and a missing folder or a taken file is refused as such, never as a failed write @@ -154,8 +149,11 @@ broke, removed by simpler code, or is equivalent: file's size before reading, so its read limit (`limit + 1`) was only reached by a file growing during the read; the read's limit is now the one check (the 25 MiB upload test kills `+ 1` → `* 1`). `openat` failed - on `fd < 0`, now on `fd == -1`, as openat(2) specifies. -- **Equivalent (MCP, all 14 missed).** `^` for `|` in the open flags + on `fd < 0`, now on `fd == -1`, as openat(2) specifies. (`inspect`, + `read_checked`, the `openat` walk and that 25 MiB test were removed + with the import roots in P2 slice 5.) +- **Equivalent (MCP, all 14 missed; the walk and its stubs were removed + in P2 slice 5).** `^` for `|` in the open flags (six: the walk's first step, each folder's two, the file's two, and `openat`'s `O_CLOEXEC`): `O_RDONLY` is 0 and `O_DIRECTORY`, `O_NOFOLLOW`, `O_NONBLOCK`, `O_CLOEXEC` are distinct bits, so `a ^ b` is `a | b`. diff --git a/docs/verification.md b/docs/verification.md index 0f6cb9d..923a720 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -238,7 +238,8 @@ was weakened. What it added is checked by tests and properties (listed in never called corrupt, damage always is; the envelope and its hash are frozen across formats, rules.md B-27), the server failing closed after a panic, the import roots' lockdown (a property over generated paths, and a -component walk that refuses a folder swapped for a link), files with full +component walk that refuses a folder swapped for a link; removed with the +import roots in P2 slice 5), files with full TINs kept out of the books (B-26), and the agent text lint. Out of scope for proof today: the format gate and the walk are I/O, and the model in `verify/lean` stops at the ledger. diff --git a/plan/served.md b/plan/served.md index b170ed8..be0738f 100644 --- a/plan/served.md +++ b/plan/served.md @@ -307,7 +307,7 @@ The books are open while something holds them; there is no background service. T - `run/mcp.sock`: served by the **server**, MCP for the bridge. Local agents never use TCP, so no other account on the machine can squat a port and catch the token or the traffic. Both ends check the peer's user ID (`SO_PEERCRED`/`getpeereid`, the pipe's SID). TCP is off by default. The owner can turn it on (`server.toml`, or `wanderbooks serve --http`) for a client that needs HTTP; it then binds 127.0.0.1:7411, a failed bind is reported by `doctor` and is not fatal, and such clients hold no lease, so the books must already be open. - `run/admin.sock`: served by the server, admin only (create, adopt, tokens), as in 3.1. Neither the viewer nor the bridge opens it. -**The server's process (P2 slice 3).** The server is the hidden mode `wanderbooks server`, started by the host with a pipe on its standard input; it runs until that input ends (the host let go of it, or died) or it gets SIGTERM, SIGINT or SIGHUP (watched from before the books are opened). It takes `run/server.lock` before anything else (a second server waits for it, touching nothing, and leaves if told to close first), opens and verifies every book (a book that won't open is listed with why, and the others are served), binds both sockets, and only then prints `ready` on its standard output. `run/mcp.sock` speaks Streamable HTTP over the Unix socket, `/b/{id}/mcp` per book, so local and remote share one code path (3.2); `run/admin.sock` takes one JSON line and answers one (`list`, `create`, `adopt`). A socket path longer than the system allows is reached through `/proc/self/fd` on Linux. Closing: new requests are refused, unrun (HTTP 503 at the door, or, for a call already past it, JSON-RPC error -32050 "closing: this call wasn't run"; the bridge sends either again once the books are back); every request already running finishes its work, however long, whether or not its client is still there; answers on their way out get 10 s more (a second signal skips that wait, never the first); then the sessions end and the sockets go. Until uploads have their own route (slice 5) a request may be 64 MiB, since receipts still ride in tool calls. A book in the root keeps its vault in its own `vault/` and its v1 key in its own `keys/legacy.key` (0700 and 0600, made private whenever first written), and no tool writes a file anywhere in the root. Adoption is a preview, then a confirm tied to the head the preview read; entries of a settings file the books don't know are refused unless skipped, its comments are shown, the vault key it names must be the one given, and the key must open every vault record. Open for slice 8: under Landlock the server can't read an adoption's source outside the root, so adoption needs another way in by then. +**The server's process (P2 slice 3).** The server is the hidden mode `wanderbooks server`, started by the host with a pipe on its standard input; it runs until that input ends (the host let go of it, or died) or it gets SIGTERM, SIGINT or SIGHUP (watched from before the books are opened). It takes `run/server.lock` before anything else (a second server waits for it, touching nothing, and leaves if told to close first), opens and verifies every book (a book that won't open is listed with why, and the others are served), binds both sockets, and only then prints `ready` on its standard output. `run/mcp.sock` speaks Streamable HTTP over the Unix socket, `/b/{id}/mcp` per book, so local and remote share one code path (3.2); `run/admin.sock` takes one JSON line and answers one (`list`, `create`, `adopt`). A socket path longer than the system allows is reached through `/proc/self/fd` on Linux. Closing: new requests are refused, unrun (HTTP 503 at the door, or, for a call already past it, JSON-RPC error -32050 "closing: this call wasn't run"; the bridge sends either again once the books are back); every request already running finishes its work, however long, whether or not its client is still there; answers on their way out get 10 s more (a second signal skips that wait, never the first); then the sessions end and the sockets go. A request is at most 1 MiB (64 MiB until slice 5 gave files their own route). A book in the root keeps its vault in its own `vault/` and its v1 key in its own `keys/legacy.key` (0700 and 0600, made private whenever first written), and no tool writes a file anywhere in the root (the server keeps uploads on their way in in the book's own `uploads/`, slice 5). Adoption is a preview, then a confirm tied to the head the preview read; entries of a settings file the books don't know are refused unless skipped, its comments are shown, the vault key it names must be the one given, and the key must open every vault record. Open for slice 8: under Landlock the server can't read an adoption's source outside the root, so adoption needs another way in by then. **The host's process (P2 slice 3).** `wanderbooks host` closes every descriptor above 2 it was handed, refuses to run confined, takes `run/host.lock` (waiting up to 30 s for a host that is closing; one that finds a live host exits at once), binds `run/lease.sock`, and starts the server from `/proc/self/exe`, naming the root on its command line only when the host itself was given one (tests): nothing in a process list names it. Ending leases, it tells each holder why and closes the server only once they have let go (at most 2 s), so holders stop sending before it closes; a call already on its way is refused at the door, unrun, and sent again once the books are back. A closing server is never killed, however long its last call runs; a new server waits for an old one still finishing (its host killed) rather than fail. A restart begins when the new version appears, not at a quiet moment: the server first finishes every call in progress (none is cut off), and calls arriving meanwhile wait for the new server (up to 60 s); waiting for idle comes with jobs (`books.job`), which are what can run long. On a restart the old host hands over: it lets go of `host.lock` and keeps `update.lock` until the new host answers, so waiting holders find it. A holder starts a host only when none holds the lock; one that died at once is noticed within 2 s; a server that fails soon after opening counts as failing (only 60 s open clears the count). Lease records carry the holder's process as the kernel reports it. The lease reasons are `closed_by_owner`, `restarting`, `closing` (the host is closing because nothing held it: hold again, a new host will answer; the owner's close, by the menu, `wanderbooks close` or a signal to the host, is always `closed_by_owner`, and a holder that had held stays closed) and `server_failed`. The bridge never sends a call the agent cancelled before it was sent (MCP owes no answer to a cancelled request), answers a ping itself while the books open, refuses JSON-RPC batches request by request, passes no client offer it can't carry back (sampling, elicitation, roots), reports a call answered with nothing as possibly run, and carries the per-request protocol headers of the 2026-07-28 era. Sessions on the server time out only after 24 h without a message (rmcp's idle timer ignores a call in progress, so a shorter one could cut a long call off); the bridge ends its own when its stdin closes or it is signalled. `wanderbooks doctor` (a first form, completed in slice 8) says whether the books are open, what holds them, and the host's and server's last log lines, naming no folder. Tests shorten the timings with a hidden `--test-knobs` flag, passed down from holder to helper to host. @@ -441,7 +441,14 @@ Report results also carry `as_of_status {drafts_open_on_or_before, unreconciled, - `export`, `asset_labels` and `export_cpa_package` return a link, plus inline JSON/CSV when ≤ 50K characters. - TIN-bearing outputs come only from `books.request_human_download`, which returns an `/h/` link. -**Typical receipt flow (Claude Code, the daily case in 7.2):** `sha256sum` → `books.begin_upload` → `curl -T` the file to `put_url` → `attach_receipt {upload_id}` → `get_receipt` (text) → `draft_from_template` or `draft_entry` → `preview_entry`, shown to the human → on "yes", `post_entry` with the `preview_token` → `next_steps` and the head come back, and `books.show` moves the viewer to the entry. No step raises a client permission prompt beyond the client's normal rules. "Save it as a draft" is `set_draft_state awaiting_evidence`. On claude.ai, files ≤ 256 KiB go inline with their sha256; larger ones go through `human_url` and the `process_inbox` prompt. +**Typical receipt flow (Claude Code, the daily case in 7.2), as planned; slice 5 built it with a command instead of `curl` (below):** `sha256sum` → `books.begin_upload` → `curl -T` the file to `put_url` → `attach_receipt {upload_id}` → `get_receipt` (text) → `draft_from_template` or `draft_entry` → `preview_entry`, shown to the human → on "yes", `post_entry` with the `preview_token` → `next_steps` and the head come back, and `books.show` moves the viewer to the entry. No step raises a client permission prompt beyond the client's normal rules. "Save it as a draft" is `set_draft_state awaiting_evidence`. On claude.ai, files ≤ 256 KiB go inline with their sha256; larger ones go through `human_url` and the `process_inbox` prompt. + +**As built (P2 slice 5), and where it departs:** +- **No path on the server's computer is read.** `books.import_local_file`, the import roots and `--import-root` are gone, and so is `set_w9`'s `w9_file_path`; a test (`no_tool_takes_a_path`) keeps any path-like parameter out, except the three outputs with full TINs (`export_cpa_package` and `get_w9` `save_to`, `vendor_1099` `save_csv_to`), which keep theirs, written only outside the books, until the viewer takes them over in slice 6 (D-28: the viewer reads and downloads; it never writes). +- **An upload, for books served from the root:** `books.begin_upload {sha256, size, mime_type, file_name, purpose: receipt|document|import|w9}` returns `{upload_id, command, expires_at, max_bytes}`. No `put_url`, `curl` or `human_url`: the server listens on a Unix socket, so the agent runs `command`, `wanderbooks upload --book ID UPLOAD FILE`, which reads its token from the keyring (`--token NAME` is in the command when the caller's token isn't the default `agent`) and PUTs the file to `/b/{book}/u/{upload}` (D-28: the agent does it all; no `/h/inbox` drop zone, no `inbox_id`; the hosted server of P9 adds a URL). The bytes stream to `uploads/.part` and are kept only if exactly the size and SHA-256 declared, by the token that began it, once (409 after); 25 MiB at most; 15 minutes; at most 16 under way per token; lapsed ones swept when the books open, whenever a grant is made, and every 10 seconds (owner, 2026-10-19: a W-9 or a file with a TIN "shouldn't just linger there forever... EVENTUALLY cleaned up is fine"). One lock per book orders every change to its uploads (grants, sweeps, a send's start and end, a take, a give-back), and a grant is written aside and renamed. Refusals carry their code (`no_such_upload`, `not_yours`, `already_sent`, `in_use`, `not_as_declared`, `stopped`), and a refused body is read and dropped (up to 25 MiB) so the client gets the answer. Every PUT to books the server holds open is in the access log (action `upload`, with the upload's id, refusals at the door included; a request for books it doesn't hold has no log to go in), written in the same blocking task as the bytes, so a client hanging up never leaves bytes kept without their record, and bytes whose record can't be written are thrown away. An upload's id is `up_` and 32 letters `a`-`p`: with no digits, the access log's TIN masking never hides one. +- **Filed once, and only when filed.** A tool takes the upload by renaming its bytes aside (so two calls never both have it), hashes them again, and spends it only once they are in the books (an import: once its rehearsal passed and the writing begins); a call refused after taking it, or one that panics, gives it back, and a server stopped mid-call gives it back when the books next open. A tool takes only an upload of its own purpose (receipt or document: `attach_receipt`, `file_document`; import: the two CSV imports; w9: `set_w9`). `existing_attachment_id` is offered only for a receipt or document, and never names a withdrawn attachment. +- **Inline:** `content_base64` (≤ 256 KiB, its length checked before it is decoded) with a required `sha256`; never both, never neither. Over stdio only inline, and no larger file comes back: stdio is the development transport now (the owner's books run released code until the cutover, `just build` building `main`), so it gets no upload route. Names and types are checked when a file is filed, whichever way it came: a name of 1 to 255 bytes, no folder in it, no control or invisible characters; a MIME type of visible ASCII. `attach_email` and `inbox_id` aren't built. `set_w9` takes `w9_upload_id` or `w9_content_base64` + `w9_sha256`, judged a PDF, PNG or JPEG by its first bytes, with an optional `w9_file_name`. A request on `/b/{book}/mcp` is at most 1 MiB now that files have their own route (it was 64 MiB). +- **Out:** `books.get_receipt {attachment_id, view: meta (default) | bytes}`: bytes inline for a file ≤ 256 KiB; a larger one answers with `download`, the command `wanderbooks download --book ID ATTACHMENT --to FOLDER`, which GETs `/b/{book}/a/{attachment}` with the agent's token (books:read; `Content-Disposition` with the name in ASCII and RFC 8187 UTF-8, `nosniff`, and the SHA-256 as an RFC 9530 `Repr-Digest`, which the command checks before it saves), into a new file beside any of that name (`name (1).pdf`), never over one: written to a hidden file and linked in whole, so the name never holds part of it. The name it saves under is the server's made plain (no folder, no leading dot, nothing invisible, at most 200 bytes), else `attachment-N`. The read and its record are one blocking task, as for uploads. The `books://receipts/{id}` resource is meta only. Not built yet: `text`, `page_image` and `link` views, `books.download_link`, and links from `export`, `asset_labels` and `export_cpa_package` (slice 6, with the viewer's downloads). ### 4.8 Drafts waiting on evidence @@ -1245,6 +1252,7 @@ These are the only open owner questions. Ask them just before the move, one at a - the scaffold: `justfile`, `scripts/`, `.claude/settings.json`'s SessionStart hook, `.codex/hooks.json`; `AGENTS.md`/`CLAUDE.md` become the 3-line pointer; - the agents' old registrations: Claude Code's stdio `wanderbooks` server from `just mcp-add` (replaced by `wanderbooks connect`), and Codex's; - old binaries: `ledger-mcp`/`ledger-gui` from `just build` or `cargo install` (the one `wanderbooks` executable replaces them); +- the `live` jj workspace (`~/.cache/wanderbooks/live-ws`) that `just build` builds the released `main` in: until the cutover these books run only released code, since the work on top of `main` drops what their stdio server needs (slice 5: files over 256 KiB only by upload, which needs the data root); - the viewer's old preferences file, eframe's `app.ron` in `$XDG_DATA_HOME/wanderbooks` (the data root's default place): the viewer moves it on its first run (ledger-gui prefs.rs), and `setup` must not refuse the default place for holding only that file (slice 8 handles it: moved to the viewer's own folder first); - the agent's own notes that name the old paths (its memory), rewritten to the new arrangement. The cutover's check adds: the project folder holds none of the above, and `wanderbooks doctor` is clean.