diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index ede3123..210fcea 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -33,7 +33,7 @@ jobs: strategy: fail-fast: false matrix: - target: [fuzz_journal_line, fuzz_entry_validate, fuzz_op_sequence, fuzz_core_serde, fuzz_store_open, fuzz_store_replay, fuzz_settings_replay, fuzz_import_csv, fuzz_config_parse] + target: [fuzz_journal_line, fuzz_entry_validate, fuzz_op_sequence, fuzz_core_serde, fuzz_store_open, fuzz_store_replay, fuzz_settings_replay, fuzz_import_csv, fuzz_config_parse, fuzz_extract_pdf, fuzz_extract_xlsx] steps: - uses: actions/checkout@v4 - uses: jdx/mise-action@v2 diff --git a/Cargo.lock b/Cargo.lock index 2b0cad1..1f48969 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3879,6 +3879,7 @@ dependencies = [ "ledger-core", "ledger-gui", "ledger-host", + "ledger-import", "ledger-mcp", "ledger-package", "ledger-store", @@ -3941,7 +3942,6 @@ dependencies = [ "egui_commonmark", "egui_extras", "egui_kittest", - "hayro 0.8.0", "image", "jiff", "ledger-config", @@ -3989,11 +3989,13 @@ dependencies = [ "chrono", "csv", "deunicode", + "hayro 0.8.0", "hayro-interpret 0.8.0", "kurbo", "ledger-core", "proptest", "qrcode", + "quick-xml 0.38.4", "rqrr", "rust_decimal", "rust_xlsxwriter", @@ -4004,6 +4006,7 @@ dependencies = [ "typst-as-lib", "typst-layout", "typst-pdf", + "zip 9.0.2", ] [[package]] @@ -4065,6 +4068,7 @@ dependencies = [ "getrandom 0.2.17", "hex", "ledger-core", + "ledger-import", "libc", "proptest", "rusqlite", diff --git a/bookspec/spec/rules-index.md b/bookspec/spec/rules-index.md index 009f591..aec30a6 100644 --- a/bookspec/spec/rules-index.md +++ b/bookspec/spec/rules-index.md @@ -56,5 +56,5 @@ without moving anyone else's number. | B-41 | `BS-arap-5` | arap | CHOICE | A debt that will not be collected is written off, and a recovery is income | `bad-debt-direct-write-off` | | B-42 | `BS-arap-6` | arap | CONVENTION | A settlement is exact, one party, and names its lines; the gross discount and the gross fee | — | | B-43 | `BS-entry-5` | entry | — | Drafts and evidence (allocated, not yet stated) | — | -| B-44 | `BS-ledger-12` | ledger | — | Extraction and dedup (allocated, not yet stated) | — | +| B-44 | `BS-ledger-12` | ledger | — | Evidence: extraction, dedup, annotations, settlements (stated in rules.md; its bookspec example comes with P13 step 5) | — | | B-45 | `BS-ledger-13` | ledger | — | Audits and attestations (allocated, not yet stated) | — | diff --git a/crates/ledger-cli/Cargo.toml b/crates/ledger-cli/Cargo.toml index aa2786d..bd2739a 100644 --- a/crates/ledger-cli/Cargo.toml +++ b/crates/ledger-cli/Cargo.toml @@ -39,6 +39,7 @@ tokio = { version = "1", features = ["macros", "rt-multi-thread", "time"] } ledger-config = { path = "../ledger-config" } ledger-core = { path = "../ledger-core" } ledger-package = { path = "../ledger-package" } +ledger-import = { path = "../ledger-import" } ledger-vault = { path = "../ledger-vault" } serde_json = "1" sha2 = "0.10" diff --git a/crates/ledger-cli/tests/evidence.rs b/crates/ledger-cli/tests/evidence.rs new file mode 100644 index 0000000..5c7eec7 --- /dev/null +++ b/crates/ledger-cli/tests/evidence.rs @@ -0,0 +1,469 @@ +//! The evidence pipeline over the wire (plan P6): a receipt's text and +//! pages, annotations, finding a file, consolidating duplicates, and the +//! backfill command. +//! +//! Expected behavior comes from `docs/mcp-interface.md` ("Tools — +//! evidence", "Tools — receipts & documents") and rules.md B-44. Every +//! call goes through a real books server as an agent reaches it +//! (`common.rs`). + +mod common; + +use common::{Books, answer_of}; +use serde_json::json; +use std::path::{Path, PathBuf}; + +/// The settings a test's books start with: a high receipt threshold (so a +/// simple posting needs no receipt) and cash at 1000. +const SETTINGS: &str = "[company]\nname = \"Evidence Co\"\nfiscal_year_start = \"01-01\"\n\ +[books]\ncash_accounts = [\"1000\"]\nreceipt_threshold = \"1000.00\"\n"; + +/// A PDF with one page that says the word "Rent" (the extractor's own +/// typesetter, so the text is real and the page renders). +fn a_pdf() -> Vec { + use ledger_import::report::{ReportMeta, Row, RowKind, statement}; + use rust_decimal::Decimal; + let st = statement( + "Receipt", + "March 2026".to_string(), + &["Total"], + vec![Row { + kind: RowKind::Line, + label: "Rent".into(), + amounts: vec![Some(Decimal::new(10000, 2))], + currency: true, + rule_above: false, + }], + ); + let meta = ReportMeta { + company: "Evidence Co".into(), + basis: "Accrual".into(), + prepared_on: chrono::NaiveDate::from_ymd_opt(2026, 3, 2).unwrap(), + journal_seq: 1, + }; + ledger_import::statement_pdf(&st, &meta).unwrap() +} + +/// A draft with two lines, and the accounts made for it. +fn a_draft(w: &mut common::Wire) -> u64 { + for (code, name, ty) in [("1000", "Cash", "asset"), ("6000", "Rent", "expense")] { + let r = w.call( + "books.add_account", + json!({"code": code, "name": name, "account_type": ty}), + ); + if let Err(e) = answer_of(&r) { + assert!(e.contains("already"), "{e}"); + } + } + let r = w.call( + "books.draft_entry", + json!({"date": "2026-03-01", "description": "March rent"}), + ); + let id = answer_of(&r).unwrap()["entry_id"].as_u64().unwrap(); + for (account, side, amount) in [("6000", "debit", "100.00"), ("1000", "credit", "100.00")] { + let r = w.call( + "books.add_line", + json!({"entry_id": id, "account": account, "side": side, "amount": amount}), + ); + answer_of(&r).unwrap(); + } + id +} + +/// File `bytes` on `entry` through an upload, as an agent does (D-46). +fn file(w: &mut common::Wire, entry: u64, bytes: &[u8], name: &str, mime: &str) -> u64 { + let upload = w.upload(bytes, name, mime, "receipt"); + let r = w.call( + "books.attach_receipt", + json!({"entry_id": entry, "upload_id": upload}), + ); + answer_of(&r).unwrap()["attachment_id"].as_u64().unwrap() +} + +/// A PNG's width and height, from its IHDR. +fn png_size(bytes: &[u8]) -> (u32, u32) { + assert!(bytes.starts_with(b"\x89PNG\r\n\x1a\n"), "not a PNG"); + let w = u32::from_be_bytes(bytes[16..20].try_into().unwrap()); + let h = u32::from_be_bytes(bytes[20..24].try_into().unwrap()); + (w, h) +} + +/// A book folder with two live receipts of one file's bytes, filed by an +/// older version on two entries (plan P6 step 4: legacy duplicates). With +/// `withdraw_first`, the first is already withdrawn, as a duplicate of the +/// second, so the second is its entry's only evidence. +fn legacy_duplicates(only: &str, withdraw_first: bool) -> (tempfile::TempDir, PathBuf) { + use ledger_core::{AccountType, Currency, Money, Side}; + let root = tempfile::Builder::new() + .prefix(&format!("wb-p6-{only}-")) + .tempdir() + .unwrap(); + let dir = root.path().join("book"); + std::fs::create_dir_all(&dir).unwrap(); + let mut s = ledger_store::Store::create(&dir, Currency::USD).unwrap(); + let cash = s + .add_account("1000", "Cash", AccountType::Asset, None, false) + .unwrap(); + let expense = s + .add_account("6000", "Rent", AccountType::Expense, None, false) + .unwrap(); + let mut entries = Vec::new(); + for (d, what) in [("2026-03-01", "Rent one"), ("2026-03-02", "Rent two")] { + let e = s + .create_entry( + chrono::NaiveDate::parse_from_str(d, "%Y-%m-%d").unwrap(), + what, + ) + .unwrap(); + let m = Money::new("100.00".parse().unwrap(), Currency::USD); + s.add_line(e, expense, Side::Debit, m).unwrap(); + s.add_line(e, cash, Side::Credit, m).unwrap(); + s.post_entry(e).unwrap(); + entries.push(e); + } + drop(s); + let bytes = b"one receipt, filed twice by an older version"; + let (sha, _) = ledger_store::BlobStore::open(&dir.join("blobs")) + .unwrap() + .put(bytes) + .unwrap(); + let mut w = ledger_store::JournalWriter::open(&dir.join(ledger_store::JOURNAL_FILE)).unwrap(); + for (id, entry, name) in [(1u64, entries[0], "one.txt"), (2, entries[1], "two.txt")] { + w.append( + "receipt.attached", + json!({ + "attachment_id": id, "blob_sha": sha, "entry_id": entry, + "category": "receipts", "file_name": name, + "mime_type": "text/plain", "size_bytes": bytes.len() as u64, + "description": null, + }), + ) + .unwrap(); + } + if withdraw_first { + w.append( + "receipt.unlinked", + json!({"attachment_id": 1, "entry_id": entries[0], "reason": "duplicate"}), + ) + .unwrap(); + w.append( + "attachment.withdrawn", + json!({"attachment_id": 1, "reason": "duplicate of #2", "duplicate_of": 2}), + ) + .unwrap(); + } + drop(w); + for f in [ + ledger_store::db_file(), + ledger_store::LOCK_FILE.to_string(), + format!("{}-wal", ledger_store::db_file()), + format!("{}-shm", ledger_store::db_file()), + ] { + let _ = std::fs::remove_file(dir.join(f)); + } + (root, dir) +} + +/// Plan P6 step 3: the default answer is what the file is, the first 8 KiB +/// of its text and its pages; never its bytes, and small enough to read. +#[test] +fn get_receipt_default_has_text_and_no_bytes() { + let books = Books::with("p6-meta", "USD", Some(SETTINGS)); + let mut w = books.session("evidence"); + let entry = a_draft(&mut w); + let pdf = a_pdf(); + let id = file(&mut w, entry, &pdf, "receipt.pdf", "application/pdf"); + let r = w.call("books.get_receipt", json!({"attachment_id": id})); + let out = answer_of(&r).unwrap(); + assert_eq!(out["text_status"], "ok", "{out}"); + assert_eq!(out["pages"], 1, "{out}"); + let text = out["text"].as_str().unwrap(); + assert!(text.contains("Rent"), "{text}"); + assert!(text.len() <= 8 * 1024); + assert_eq!(out["text_truncated"], false); + assert!(out.get("file").is_none(), "no file in the default answer"); + assert!( + serde_json::to_string(&out).unwrap().len() < 8 * 1024, + "the default answer is under 8 KiB" + ); + assert!(!serde_json::to_string(&out).unwrap().contains("base64")); + // view text: the whole text, no file. + let r = w.call( + "books.get_receipt", + json!({"attachment_id": id, "view": "text"}), + ); + let out = answer_of(&r).unwrap(); + assert_eq!(out["text"].as_str().unwrap(), text); + // The resource reads the same: metadata plus text. + let r = w.rpc( + "resources/read", + json!({"uri": format!("books://attachments/{id}")}), + ); + let text = r["result"]["contents"][0]["text"].as_str().unwrap(); + assert!(text.contains("\"attachment_id\""), "{text}"); + assert!(text.contains("Rent"), "{text}"); +} + +/// Plan P6 step 3: one page as a PNG, handed over; page numbers start at +/// 1; a page past the last is refused. +#[test] +fn page_image_is_a_png_of_the_page() { + let books = Books::with("p6-page", "USD", Some(SETTINGS)); + let mut w = books.session("evidence"); + let entry = a_draft(&mut w); + let id = file(&mut w, entry, &a_pdf(), "receipt.pdf", "application/pdf"); + let r = w.call( + "books.get_receipt", + json!({"attachment_id": id, "view": "page_image", "page": 1, "max_px": 300}), + ); + let out = answer_of(&r).unwrap(); + let file = &out["file"]; + assert_eq!(file["mime_type"], "image/png"); + let png = w.fetch(file); + let (width, height) = png_size(&png); + assert!(width.max(height) <= 300, "{width}x{height}"); + assert!(width > 0 && height > 0); + // The page resource gives a PNG of the page too (its size its own: + // a resource takes no max_px). + let r = w.rpc( + "resources/read", + json!({"uri": format!("books://attachments/{id}/page/1")}), + ); + let contents = &r["result"]["contents"][0]; + assert_eq!(contents["mimeType"], "image/png", "{contents}"); + let encoded = contents["blob"].as_str().unwrap(); + use base64::Engine; + let bytes = base64::engine::general_purpose::STANDARD + .decode(encoded) + .unwrap(); + png_size(&bytes); + // Past the last page: refused. + let r = w.call( + "books.get_receipt", + json!({"attachment_id": id, "view": "page_image", "page": 2}), + ); + assert!(answer_of(&r).is_err()); + // bytes still hands the file over, unchanged. + let r = w.call( + "books.get_receipt", + json!({"attachment_id": id, "view": "bytes"}), + ); + let out = answer_of(&r).unwrap(); + assert_eq!(w.fetch(&out["file"]), a_pdf()); +} + +/// Plan P6 steps 4 and 5: annotations, the engine's numbers, the three +/// ways to find a file, and a settlement recorded. +#[test] +fn annotations_and_searches() { + let books = Books::with("p6-find", "USD", Some(SETTINGS)); + let mut w = books.session("evidence"); + let entry = a_draft(&mut w); + let text = b"Harbor Design LLC\nInvoice H-1042\nHosting, 42.00, 2026-03-01\n"; + let id = file(&mut w, entry, text, "harbor.txt", "text/plain"); + let r = w.call( + "books.annotate_attachment", + json!({"attachment_id": id, "fields": { + "receipt_number": 7, "payee": "Harbor Design LLC", "amount": "42.00", + "date": "2026-03-01", "instrument": "Check 1042", + "source": "the owner's email", "reimbursable_to": "Avery Quill" + }, "note": "read on 2026-03-02"}), + ); + let out = answer_of(&r).unwrap(); + assert_eq!(out["fields"]["receipt_number"], 7); + assert_eq!(out["fields"]["amount"], "42.00"); + assert_eq!(out["next_receipt_number"], 8); + // Meta carries the annotation and the number. + let r = w.call("books.get_receipt", json!({"attachment_id": id})); + let out = answer_of(&r).unwrap(); + assert_eq!(out["receipt_number"], 7); + assert_eq!(out["annotation"]["payee"], "Harbor Design LLC"); + // Found three ways. + for args in [ + json!({"receipt_number": 7}), + json!({"sha256": ledger_store::journal::sha256_hex(text)}), + json!({"text": "harbor design"}), + ] { + let r = w.call("books.find_attachment", args.clone()); + let out = answer_of(&r).unwrap(); + assert!( + out["attachments"] + .as_array() + .unwrap() + .iter() + .any(|a| a["attachment_id"] == id), + "{args}: {out}" + ); + } + // Exactly one selector. + let r = w.call( + "books.find_attachment", + json!({"sha256": ledger_store::journal::sha256_hex(text), "text": "x"}), + ); + assert!(answer_of(&r).is_err()); + // A settlement is recorded on the paying entry and read back. + let pay = a_draft(&mut w); + let r = w.call( + "books.set_entry_settles", + json!({"entry_id": pay, "settles": [entry]}), + ); + let out = answer_of(&r).unwrap(); + assert_eq!(out["settles"], json!([entry])); + let r = w.call( + "books.set_entry_settles", + json!({"entry_id": pay, "settles": [pay]}), + ); + assert!(answer_of(&r).is_err(), "an entry settles not itself"); + let r = w.call("books.verify", json!({})); + assert_eq!(answer_of(&r).unwrap()["status"], "ok"); +} + +/// Plan P6 step 4 and rules.md B-44 item 5: consolidation never leaves an +/// entry the receipt policy covers with no evidence. +#[test] +fn consolidation_never_removes_an_entrys_only_evidence() { + let (_root, fixture) = legacy_duplicates("dupes-only", true); + let books = Books::placed("p6-dupes-only", &fixture); + let mut w = books.session("evidence"); + let before = books.journal(); + let r = w.call("books.consolidate_duplicates", json!({"confirmed": true})); + let why = answer_of(&r).unwrap_err(); + assert!(why.contains("receipt_required"), "{why}"); + assert_eq!(books.journal(), before, "a refusal journals nothing"); +} + +/// Plan P6 step 4: the kept file takes every subject the duplicates had; +/// the others are withdrawn as duplicates of it, and nothing is deleted. +#[test] +fn consolidation_links_the_kept_file_to_every_subject() { + let (_root, fixture) = legacy_duplicates("dupes-live", false); + let books = Books::placed("p6-dupes-live", &fixture); + let mut w = books.session("evidence"); + let sha = { + let r = w.call("books.get_receipt", json!({"attachment_id": 2})); + answer_of(&r).unwrap()["sha256"] + .as_str() + .unwrap() + .to_string() + }; + let r = w.call("books.consolidate_duplicates", json!({})); + let preview = answer_of(&r).unwrap(); + assert_eq!(preview["status"], "preview"); + assert_eq!(preview["groups"][0]["keep"], 1); + assert_eq!(preview["groups"][0]["withdraw"], json!([2])); + let before = books.journal(); + assert_eq!(before, books.journal(), "a preview journals nothing"); + let r = w.call("books.consolidate_duplicates", json!({"confirmed": true})); + let out = answer_of(&r).unwrap(); + assert_eq!(out["status"], "consolidated", "{out}"); + assert_eq!(out["withdrawn"], 1); + // The second entry is still evidenced, by the kept file. + let r = w.call("books.list_receipts", json!({"entry_id": 2})); + let out = answer_of(&r).unwrap(); + assert_eq!(out["receipts"].as_array().unwrap().len(), 1, "{out}"); + assert_eq!(out["receipts"][0]["attachment_id"], 1); + // The withdrawn one is found, marked, and names what it duplicates. + let r = w.call("books.find_attachment", json!({"sha256": sha})); + let out = answer_of(&r).unwrap(); + let found = out["attachments"].as_array().unwrap(); + assert_eq!(found.len(), 2, "{out}"); + let gone = found.iter().find(|a| a["attachment_id"] == 2).unwrap(); + assert_eq!(gone["withdrawn"], true); + let r = w.call("books.verify", json!({})); + assert_eq!(answer_of(&r).unwrap()["status"], "ok"); +} + +/// Done when (1): every attachment of the `p2-served` fixture gives a +/// default `books.get_receipt` answer under 8 KiB. +#[test] +fn every_p2_served_attachment_has_a_small_default_answer() { + let fixture = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/p2-served"); + let books = Books::placed("p6-p2-served", &fixture); + let mut w = books.session("evidence"); + let mut seen = 0; + for id in 1..=64u64 { + let r = w.call("books.get_receipt", json!({"attachment_id": id})); + let Ok(out) = answer_of(&r) else { continue }; + seen += 1; + let size = serde_json::to_string(&out).unwrap().len(); + assert!(size < 8 * 1024, "attachment {id}: {size} bytes"); + assert_eq!(out["text_truncated"], false, "attachment {id}"); + assert!(out.get("file").is_none(), "attachment {id}"); + } + assert!(seen >= 60, "the fixture's attachments: {seen}"); +} + +/// Done when (4): the backfill reads an older book's files, one +/// attachment at a time, and a second run journals nothing. +#[test] +fn the_backfill_reads_an_old_fixtures_files_once() { + let fixture = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../ledger-store/tests/fixtures/old-books/p6_evidence"); + let books = Books::placed("p6-backfill", &fixture); + // The fixture is from before P6: upgrade it (the owner's step), then + // the backfill can journal extraction records. + books.stop(); + { + let mut s = ledger_store::Store::open(&books.dir()).unwrap(); + s.upgrade_format().unwrap(); + s.verify().unwrap(); + } + books.start(); + let mut w = books.session("evidence"); + // Attachment 1 is a PDF (one page with text), 2 a text file, 3 a + // Markdown document. + for id in 1..=2 { + let before = books.journal(); + let r = w.call("books.get_receipt", json!({"attachment_id": id})); + let out = answer_of(&r).unwrap(); + assert_eq!(out["text_status"], "not_extracted", "{out}"); + let answer = books.home.admin(ledger_mcp::admin::Request::Extract { + id: books.id.clone(), + attachment_id: id, + }); + match answer { + ledger_mcp::admin::Response::Extracted { status, .. } => assert_eq!(status, "ok"), + other => panic!("{other:?}"), + } + assert_ne!(books.journal(), before, "the backfill journals a record"); + } + // The PDF's text is readable now, and its pages render. + let r = w.call("books.get_receipt", json!({"attachment_id": 1})); + let out = answer_of(&r).unwrap(); + assert_eq!(out["text_status"], "ok", "{out}"); + assert_eq!(out["pages"], 1, "{out}"); + assert!(out["text"].as_str().unwrap().contains("Rent"), "{out}"); + // The command itself, on the Markdown document (attachment 3), and + // again: a second run journals nothing. The test's own server is + // stopped first: the command opens the books and closes them as the + // host does. + books.stop(); + let run = || { + std::process::Command::new(env!("CARGO_BIN_EXE_wanderbooks")) + .args(["books", "extract", &books.id, "3", "--root"]) + .arg(books.home.root()) + .env_clear() + .env("PATH", "/usr/bin:/bin") + .env("HOME", books.home.dir.path()) + .output() + .unwrap() + }; + let before = books.journal(); + let out = run(); + assert!(out.status.success(), "{out:?}"); + assert!( + String::from_utf8_lossy(&out.stdout).contains("Read attachment 3"), + "{out:?}" + ); + assert_ne!(books.journal(), before, "the command journals a record"); + let after = books.journal(); + let out = run(); + assert!(out.status.success(), "{out:?}"); + assert_eq!(books.journal(), after, "an extract journals once"); + books.start(); + let mut w = books.session("evidence"); + let r = w.call("books.verify", json!({})); + assert_eq!(answer_of(&r).unwrap()["status"], "ok"); + let r = w.call("books.get_receipt", json!({"attachment_id": 3})); + assert_eq!(answer_of(&r).unwrap()["text_status"], "ok"); +} diff --git a/crates/ledger-cli/tests/format_upgrade.rs b/crates/ledger-cli/tests/format_upgrade.rs index 417028d..e68d865 100644 --- a/crates/ledger-cli/tests/format_upgrade.rs +++ b/crates/ledger-cli/tests/format_upgrade.rs @@ -854,6 +854,13 @@ fn books_p2_served_cut_over_whole() { "reason": "collection exhausted", "existing_attachment_id": 7, "confirmed": true}), ), + // A file's annotation (attachment_text, P6): an older version + // doesn't know the event. + ( + "books.annotate_attachment", + json!({"attachment_id": 1, "fields": {"payee": "Harbor Design LLC", + "amount": "12.34", "date": "2026-02-01"}}), + ), ]; for (tool, args) in &refused { let e = a.call(tool, args.clone()).unwrap_err(); diff --git a/crates/ledger-cli/tests/mcp_journeys.rs b/crates/ledger-cli/tests/mcp_journeys.rs index acfae8e..fc812b9 100644 --- a/crates/ledger-cli/tests/mcp_journeys.rs +++ b/crates/ledger-cli/tests/mcp_journeys.rs @@ -125,7 +125,7 @@ fn mcp_end_to_end() { // Tool listing: the narrow tools; chart is read-only. let list = mcp.request("tools/list", json!({})); let tools = list["result"]["tools"].as_array().unwrap(); - assert_eq!(tools.len(), 125, "expected 125 tools, got {}", tools.len()); + assert_eq!(tools.len(), 129, "expected 129 tools, got {}", tools.len()); // Files come in as tool input: books.begin_upload, never a local path. assert!(tools.iter().any(|t| t["name"] == "books.begin_upload")); assert!(!tools.iter().any(|t| t["name"] == "books.import_local_file")); diff --git a/crates/ledger-cli/tests/oracle_mcp.rs b/crates/ledger-cli/tests/oracle_mcp.rs index 59f13f4..bf5b2e5 100644 --- a/crates/ledger-cli/tests/oracle_mcp.rs +++ b/crates/ledger-cli/tests/oracle_mcp.rs @@ -2868,7 +2868,7 @@ fn receipts_and_documents_round_trip() { json!({"attachment_id": a["attachment_id"], "view": view}), ); assert!( - why.contains("meta (the default) or bytes"), + why.contains("meta (the default), text, page_image or bytes"), "{view:?}: {why}" ); } diff --git a/crates/ledger-cli/tests/spec_mcp.rs b/crates/ledger-cli/tests/spec_mcp.rs index 3f02cef..b6630af 100644 --- a/crates/ledger-cli/tests/spec_mcp.rs +++ b/crates/ledger-cli/tests/spec_mcp.rs @@ -822,6 +822,37 @@ fn happy_path(name: &str) -> Mcp { ); assert_eq!(up["existing_attachment_id"], json!(local), "{up}"); + // P6: the evidence pipeline. Find the local receipt by its text, give + // it a receipt number, settle nothing, and consolidate (there are no + // duplicates here: a preview and a no-op confirm still exercise both + // answers). + let found = m.ok("books.find_attachment", json!({"text": "local receipt"})); + assert_eq!( + found["attachments"][0]["attachment_id"], + json!(local), + "{found}" + ); + let annotated = m.ok( + "books.annotate_attachment", + json!({"attachment_id": local, "fields": {"receipt_number": 1, + "payee": "Landlord", "amount": "12.34", "date": "2026-01-10"}}), + ); + assert_eq!(annotated["next_receipt_number"], 2, "{annotated}"); + m.ok( + "books.find_attachment", + json!({"receipt_number": annotated["fields"]["receipt_number"]}), + ); + m.ok( + "books.find_attachment", + json!({"sha256": ledger_store::journal::sha256_hex(b"a local receipt")}), + ); + m.ok( + "books.set_entry_settles", + json!({"entry_id": e1, "settles": []}), + ); + m.ok("books.consolidate_duplicates", json!({})); + m.ok("books.consolidate_duplicates", json!({"confirmed": true})); + // Reversal: preview, then confirmed. let e3 = m.draft( "2026-01-20", diff --git a/crates/ledger-core/src/error.rs b/crates/ledger-core/src/error.rs index 5d3f9b7..e99bc13 100644 --- a/crates/ledger-core/src/error.rs +++ b/crates/ledger-core/src/error.rs @@ -72,6 +72,11 @@ pub enum LedgerError { #[error("unknown account id {0}")] UnknownAccount(u64), + /// What the books record about their evidence (an annotation, a + /// settlement) is not a record they keep (plan P6, [`crate::evidence`]). + #[error("{0}")] + InvalidEvidence(String), + /// Placeholder (header) accounts organize the chart; they take no postings. #[error("account '{0}' is a placeholder and cannot take postings")] PlaceholderAccount(String), diff --git a/crates/ledger-core/src/evidence.rs b/crates/ledger-core/src/evidence.rs new file mode 100644 index 0000000..465add3 --- /dev/null +++ b/crates/ledger-core/src/evidence.rs @@ -0,0 +1,120 @@ +//! What the books record about their evidence beyond the file itself +//! (plan P6 step 5): a receipt's own facts (its number, payee, amount, +//! date, instrument, where it came from, who is to be reimbursed) and +//! which entries a payment settles. +//! +//! ```text +//! GOAL: an annotation is a record of what a human read off a file, kept +//! apart from the file and the ledger, and a settlement is the one +//! record from which "the member was paid back" is derived. +//! SOURCE: plan/served.md P6 step 5 (defect 10); IZ-18 (no FIFO guess: +//! `entry.settles` only); IRS Pub. 583 (records are kept with what +//! they support). +//! INVARIANTS: +//! 1. An annotation's fields are a closed list; every one is optional, +//! and at least one field or the note is present. +//! 2. A receipt number runs from the highest one annotated +//! ([`crate::Ledger::next_receipt_number`]). +//! 3. A settlement names existing entries, never itself, never one +//! twice; it records nothing about amounts. +//! ``` + +use chrono::NaiveDate; +use rust_decimal::Decimal; + +use crate::LedgerError; + +/// The closed set of facts a human may record about a file (plan P6 step +/// 5). Every field is optional; the amount is an exact decimal in the +/// books' own currency, and the date is the date on the document. +#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct AnnotationFields { + /// The number printed on the receipt, unique in its own series only. + pub receipt_number: Option, + /// Who was paid. + pub payee: Option, + /// The amount printed on the receipt, exact. + pub amount: Option, + /// The date printed on the receipt. + pub date: Option, + /// How it was paid: a payment method's id or label. + pub instrument: Option, + /// What kind of record it is, as the document categories name them. + pub category: Option, + /// Where it came from (a folder, a person, an email printed to PDF). + pub source: Option, + /// Who is to be reimbursed for it, if anyone. + pub reimbursable_to: Option, +} + +impl AnnotationFields { + /// Whether nothing at all is recorded. + pub fn is_empty(&self) -> bool { + self.receipt_number.is_none() + && self.payee.is_none() + && self.amount.is_none() + && self.date.is_none() + && self.instrument.is_none() + && self.category.is_none() + && self.source.is_none() + && self.reimbursable_to.is_none() + } + + /// Check the fields before they are journaled (INVARIANTS 1 and 2): + /// every text trimmed and non-empty, the number positive, the amount + /// not negative, the date in the books' calendar. Returns them as they + /// will be kept. + pub fn checked(mut self) -> Result { + let trim = |s: Option| -> Option { + s.map(|s| s.trim().to_string()).filter(|s| !s.is_empty()) + }; + self.payee = trim(self.payee); + self.instrument = trim(self.instrument); + self.category = trim(self.category); + self.source = trim(self.source); + self.reimbursable_to = trim(self.reimbursable_to); + if self.receipt_number == Some(0) { + return Err(LedgerError::InvalidEvidence( + "a receipt number is positive".into(), + )); + } + if self.amount.is_some_and(|a| a.is_sign_negative()) { + return Err(LedgerError::InvalidEvidence( + "an annotated amount is not negative".into(), + )); + } + if let Some(d) = self.date { + crate::calendar::check(d)?; + } + Ok(self) + } +} + +/// One recorded annotation: the file, the facts read off it, and an +/// optional note. +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct Annotation { + /// The attachment the facts are about. + pub attachment_id: u64, + /// The facts. + pub fields: AnnotationFields, + /// A free note. + pub note: Option, +} + +impl Annotation { + /// Check the annotation before it is journaled (INVARIANTS 1 and 2). + pub fn checked(mut self) -> Result { + self.fields = self.fields.checked()?; + self.note = self + .note + .map(|n| n.trim().to_string()) + .filter(|n| !n.is_empty()); + if self.fields.is_empty() && self.note.is_none() { + return Err(LedgerError::InvalidEvidence( + "an annotation records at least one field or a note".into(), + )); + } + Ok(self) + } +} diff --git a/crates/ledger-core/src/ledger.rs b/crates/ledger-core/src/ledger.rs index 18edd80..8834994 100644 --- a/crates/ledger-core/src/ledger.rs +++ b/crates/ledger-core/src/ledger.rs @@ -349,6 +349,12 @@ pub struct Ledger { /// Conventions, decisions, open items and approvals /// ([`crate::conventions`]). conventions: crate::conventions::Conventions, + /// The facts recorded about each filed file ([`crate::evidence`], + /// `attachment.annotated`), by attachment id. + annotations: std::collections::BTreeMap, + /// What each entry settles (`entry.settles`, plan P6): the one record + /// a member-paid entry's "reimbursed" is derived from (IZ-18). + entry_settles: std::collections::BTreeMap>, } impl Ledger { @@ -403,6 +409,8 @@ impl Ledger { entry_no_party: HashMap::new(), settings: crate::settings::Settings::default(), conventions: crate::conventions::Conventions::default(), + annotations: std::collections::BTreeMap::new(), + entry_settles: std::collections::BTreeMap::new(), } } @@ -816,6 +824,8 @@ impl Ledger { ), ("settings", self.settings == other.settings), ("conventions", self.conventions == other.conventions), + ("annotations", self.annotations == other.annotations), + ("entry_settles", self.entry_settles == other.entry_settles), ] .into_iter() .filter(|(_, same)| !same) @@ -8334,6 +8344,101 @@ impl Ledger { &self.conventions } + /// The facts recorded about the books' files, by attachment id + /// (`attachment.annotated`, plan P6 step 5). + pub fn annotations(&self) -> impl Iterator { + self.annotations.values() + } + + /// The facts recorded about one file, if any. + pub fn annotation(&self, attachment_id: u64) -> Option<&crate::evidence::Annotation> { + self.annotations.get(&attachment_id) + } + + /// The next receipt number: one past the highest one annotated, or 1 + /// when none is (plan P6 step 5: engine numbers run from the max). + pub fn next_receipt_number(&self) -> u32 { + self.annotations + .values() + .filter_map(|a| a.fields.receipt_number) + .max() + .map_or(1, |n| n.saturating_add(1)) + } + + /// Record what a human read off a file, replacing an earlier + /// annotation of it (plan P6 step 5; INVARIANTS 1 and 2 of + /// [`crate::evidence`]). + pub fn annotate_attachment( + &mut self, + annotation: crate::evidence::Annotation, + ) -> Result<(), LedgerError> { + let annotation = annotation.checked()?; + self.annotations + .insert(annotation.attachment_id, annotation); + Ok(()) + } + + /// [`Self::annotate_attachment`] as journaled (`attachment.annotated`), + /// for replay only (rules.md B-28). + pub fn replay_annotate_attachment( + &mut self, + annotation: crate::evidence::Annotation, + ) -> Result<(), LedgerError> { + self.annotate_attachment(annotation) + } + + /// The entries `entry` records settling (`entry.settles`, plan P6). + pub fn entry_settles(&self, entry: EntryId) -> &[EntryId] { + self.entry_settles.get(&entry).map_or(&[], Vec::as_slice) + } + + /// Whether any entry records settling `entry`: the one way a + /// member-paid entry reads as reimbursed (IZ-18: no FIFO guess). + pub fn settles_entry(&self, entry: EntryId) -> bool { + self.entry_settles.values().any(|s| s.contains(&entry)) + } + + /// Record which entries `entry` settles (a reimbursement naming what + /// it pays back, plan P6 step 5). INVARIANT 3: every entry exists, + /// none is `entry` itself, none is named twice. + pub fn set_entry_settles( + &mut self, + entry: EntryId, + settles: Vec, + ) -> Result<(), LedgerError> { + if !self.entries.contains_key(&entry) { + return Err(LedgerError::UnknownEntry(entry)); + } + let mut seen = std::collections::BTreeSet::new(); + for s in &settles { + if *s == entry { + return Err(LedgerError::InvalidEvidence( + "an entry doesn't settle itself".into(), + )); + } + if !self.entries.contains_key(s) { + return Err(LedgerError::UnknownEntry(*s)); + } + if !seen.insert(*s) { + return Err(LedgerError::InvalidEvidence(format!( + "entry {s} is settled twice by entry {entry}" + ))); + } + } + self.entry_settles.insert(entry, settles); + Ok(()) + } + + /// [`Self::set_entry_settles`] as journaled (`entry.settles`), for + /// replay only (rules.md B-28). + pub fn replay_set_entry_settles( + &mut self, + entry: EntryId, + settles: Vec, + ) -> Result<(), LedgerError> { + self.set_entry_settles(entry, settles) + } + /// Check a convention before it is journaled: its id, text and /// structure (crate::conventions::check_new), and that every account a /// posting template or approval rule names is in the chart (a diff --git a/crates/ledger-core/src/lib.rs b/crates/ledger-core/src/lib.rs index 8a00da4..85ccbb9 100644 --- a/crates/ledger-core/src/lib.rs +++ b/crates/ledger-core/src/lib.rs @@ -25,6 +25,7 @@ pub mod cashflow; pub mod classes; pub mod conventions; pub mod error; +pub mod evidence; pub mod form1099; pub mod instruments; pub mod journal; @@ -69,6 +70,7 @@ pub use cashflow::{ }; pub use classes::{Class, ClassId, ProjectFacts, ProjectKind, ProjectStatus}; pub use error::LedgerError; +pub use evidence::{Annotation, AnnotationFields}; pub use form1099::{ Dated1099, Decision1099, ExemptReason, Form1099, Payment1099, PaymentAccounts, Status1099, Vendor1099Line, Vendor1099Report, vendor_1099_report, diff --git a/crates/ledger-gui/Cargo.toml b/crates/ledger-gui/Cargo.toml index 3ef75a4..6229a84 100644 --- a/crates/ledger-gui/Cargo.toml +++ b/crates/ledger-gui/Cargo.toml @@ -24,7 +24,6 @@ egui-phosphor = "0.14" # Markdown documents, drawn as text. No image loading (no `load-images`, # no `fetch`): a document never makes the viewer read a file or the network. egui_commonmark = { version = "0.25", default-features = false, features = ["pulldown_cmark"] } -hayro = { version = "0.8", features = ["embed-fonts"] } image = { version = "0.25", default-features = false, features = ["png", "jpeg", "webp"] } notify = "8" jiff = "0.2" diff --git a/crates/ledger-gui/src/attention.rs b/crates/ledger-gui/src/attention.rs index 8331750..258f24b 100644 --- a/crates/ledger-gui/src/attention.rs +++ b/crates/ledger-gui/src/attention.rs @@ -100,6 +100,10 @@ impl ledger_package::PackageBooks for View<'_> { Ok(self.0.attachments.clone()) } + fn extractions(&self) -> Result, StoreError> { + Ok(self.0.extractions.clone()) + } + fn links_of(&self, attachment_id: u64) -> Result, StoreError> { Ok(self.0.links_of(attachment_id).to_vec()) } diff --git a/crates/ledger-gui/src/books.rs b/crates/ledger-gui/src/books.rs index 4fe24d8..9f8a950 100644 --- a/crates/ledger-gui/src/books.rs +++ b/crates/ledger-gui/src/books.rs @@ -325,6 +325,9 @@ pub struct Books { pub statements: Vec, /// Reconciliations, voided ones included, by date then id. pub reconciliations: Vec, + /// What reading each file's text found (plan P6): the same records the + /// server's Needs attention reads. + pub extractions: Vec, /// The format upgrade the books wait for, if they do (plan 3.9, D-39): /// read from their journal, as the server reads it. pub format_upgrade: Option, @@ -666,6 +669,7 @@ impl Books { } let statements = snap.statements(None).map_err(|e| e.to_string())?; let reconciliations = snap.reconciliations(None).map_err(|e| e.to_string())?; + let extractions = snap.extractions().map_err(|e| e.to_string())?; let fresh_from_seq = prev_seq.unwrap_or(journal.len() as u64); let mut fresh = HashSet::new(); let closings = ledger.closing_entry_ids(); @@ -832,6 +836,7 @@ impl Books { links, statements, reconciliations, + extractions, format_upgrade: snap.journal_format().upgrade(), }; books.parties = books.party_infos(); diff --git a/crates/ledger-gui/src/pages/documents.rs b/crates/ledger-gui/src/pages/documents.rs index 3c58fe3..2b2498d 100644 --- a/crates/ledger-gui/src/pages/documents.rs +++ b/crates/ledger-gui/src/pages/documents.rs @@ -18,6 +18,9 @@ use ledger_store::{Attachment, Subject}; #[derive(Default)] pub struct State { pub category: Option, + /// One subject's files only (plan P6 step 6: grouped by category and + /// subject). + pub subject: Option, pub show_withdrawn: bool, pub query: String, } @@ -72,6 +75,38 @@ fn grid(ui: &mut Ui, b: &Books, selected: Option, shell: &mut Shell<'_>) { st.category = Some(c.clone()); } } + // One subject's files (plan P6 step 6): every subject any + // document is evidence of, and the company's own. + let mut subjects: Vec = Vec::new(); + for a in &b.attachments { + if let Some(e) = a.entry_id { + subjects.push(Subject::Entry(e)); + } + subjects.extend(b.links_of(a.id)); + } + subjects.sort(); + subjects.dedup(); + egui::ComboBox::from_id_salt("documents-subject") + .selected_text(match st.subject { + None => "Any subject".to_string(), + Some(s) => b.subject_label(s), + }) + .show_ui(ui, |ui| { + if ui + .selectable_label(st.subject.is_none(), "Any subject") + .clicked() + { + st.subject = None; + } + for s in subjects { + if ui + .selectable_label(st.subject == Some(s), b.subject_label(s)) + .clicked() + { + st.subject = Some(s); + } + } + }); ui.checkbox(&mut st.show_withdrawn, "Show withdrawn"); }); ui.add_space(12.0); @@ -82,6 +117,12 @@ fn grid(ui: &mut Ui, b: &Books, selected: Option, shell: &mut Shell<'_>) { .rev() .filter(|a| st.show_withdrawn || a.withdrawn_at.is_none()) .filter(|a| st.category.as_ref().is_none_or(|c| &a.category == c)) + .filter(|a| match st.subject { + None => true, + Some(s) => { + a.entry_id.map(Subject::Entry) == Some(s) || b.links_of(a.id).contains(&s) + } + }) .filter(|a| { q.is_empty() || a.file_name.to_lowercase().contains(&q) @@ -101,16 +142,34 @@ fn grid(ui: &mut Ui, b: &Books, selected: Option, shell: &mut Shell<'_>) { let tile_w = ((ui.available_width() - 16.0 * (per_row as f32 - 1.0)) / per_row as f32).min(280.0); let mut go = None; - for row in docs.chunks(per_row) { - ui.horizontal_top(|ui| { - ui.spacing_mut().item_spacing.x = 16.0; - for a in row { - if tile(ui, b, a, tile_w, selected == Some(a.id), shell).clicked() { - go = Some(a.id); + // Grouped by category unless one is chosen (plan P6 step 6): a + // heading, then its files; the subject filter narrows each group. + let by_category = st.category.is_none(); + let mut groups: std::collections::BTreeMap> = + std::collections::BTreeMap::new(); + for a in docs { + groups.entry(a.category.clone()).or_default().push(a); + } + for (category, files) in groups { + if by_category { + ui.label( + RichText::new(fmt::label(&category)) + .font(theme::semibold(12.5)) + .color(pal(ui.ctx()).muted), + ); + ui.add_space(6.0); + } + for row in files.chunks(per_row) { + ui.horizontal_top(|ui| { + ui.spacing_mut().item_spacing.x = 16.0; + for a in row { + if tile(ui, b, a, tile_w, selected == Some(a.id), shell).clicked() { + go = Some(a.id); + } } - } - }); - ui.add_space(16.0); + }); + ui.add_space(16.0); + } } if let Some(id) = go { shell.go(Route::Document(id)); @@ -625,6 +684,40 @@ mod tests { use super::*; use crate::test_support::{Build, d, find, render}; + /// The list groups by category and offers one subject's files (plan P6 + /// step 6). + #[test] + fn documents_group_by_category_and_subject() { + let mut b = Build::new("documents-groups"); + let (cash, capital) = (b.cash, b.capital); + let e = b.post(d(2026, 1, 2), cash, capital, 100_000); + let (accounting, _, _) = + b.s.file_document( + "accounting", + "ledger.pdf", + "application/pdf", + b"%PDF ledger", + None, + ) + .unwrap(); + b.s.link_attachment(accounting, ledger_store::Subject::Entry(e)) + .unwrap(); + b.s.file_document( + "notices", + "notice.pdf", + "application/pdf", + b"%PDF notice", + None, + ) + .unwrap(); + let (_dir, books) = b.books(); + let (texts, _) = render(books, Route::Documents, |_| {}); + for group in ["Receipts", "Accounting", "Notices"] { + find(&texts, group); + } + find(&texts, "Any subject"); + } + /// A document's page lists what it is linked to: an entry with its date /// and amount. #[test] diff --git a/crates/ledger-gui/src/pages/journal.rs b/crates/ledger-gui/src/pages/journal.rs index f7c5cb6..51614e4 100644 --- a/crates/ledger-gui/src/pages/journal.rs +++ b/crates/ledger-gui/src/pages/journal.rs @@ -391,9 +391,63 @@ pub fn describe(b: &Books, line: &JournalLine) -> Option { } "attachment.withdrawn" => ev( icon::ARCHIVE, - format!("Withdrew {}: {}", attachment_name(b, att), s(p, "reason")), + format!( + "Withdrew {}{}", + attachment_name(b, att), + match p.get("duplicate_of").and_then(|v| v.as_u64()) { + Some(first) => format!(", a duplicate of #{first}: {}", s(p, "reason")), + None => format!(": {}", s(p, "reason")), + } + ), Some(Route::Document(att)), ), + "attachment.text_extracted" => ev( + icon::FILE_TEXT, + format!( + "Read {}: {}", + attachment_name(b, att), + match s(p, "status") { + "ok" => "its text".to_string(), + "none" => "no text to read".to_string(), + _ => "the reader failed".to_string(), + } + ), + Some(Route::Document(att)), + ), + "attachment.annotated" => ev( + icon::PENCIL_SIMPLE, + match p + .get("fields") + .and_then(|f| f.get("receipt_number")) + .and_then(|v| v.as_u64()) + { + Some(n) => format!("Annotated {} as receipt {n}", attachment_name(b, att)), + None => format!("Annotated {}", attachment_name(b, att)), + }, + Some(Route::Document(att)), + ), + "entry.settles" => { + let settles: Vec = p + .get("settles") + .and_then(|v| v.as_array()) + .map(|a| { + a.iter() + .filter_map(|v| v.as_u64()) + .map(|e| format!("#{e}")) + .collect() + }) + .unwrap_or_default(); + let text = if settles.is_empty() { + format!("Cleared what #{} settles", entry_id.unwrap_or(0)) + } else { + format!( + "Entry #{} settles {}", + entry_id.unwrap_or(0), + settles.join(", ") + ) + }; + ev(icon::CHECK, text, entry_id.map(Route::Entry)) + } "account.added" => ev( icon::PLUS_CIRCLE, format!("Added account {} {}", s(p, "code"), s(p, "name")), diff --git a/crates/ledger-gui/src/receipts.rs b/crates/ledger-gui/src/receipts.rs index 52e957c..e4e3822 100644 --- a/crates/ledger-gui/src/receipts.rs +++ b/crates/ledger-gui/src/receipts.rs @@ -1,7 +1,7 @@ //! Rendering receipts and documents to textures, off the UI thread. //! -//! PDFs are rasterized with `hayro` (pure Rust), images decoded with -//! `image`; Markdown and plain text are read as UTF-8 and drawn as text +//! PDFs are rasterized by the one extractor in `ledger-import` (pure Rust, +//! plan P6), images decoded with `image`; Markdown and plain text are read as UTF-8 and drawn as text //! (Markdown by `egui_commonmark`), so they follow the theme and select //! like any text. Results are cached by blob hash: the same bytes never render //! twice, and a hash is the content, so the cache never goes stale. Bytes @@ -61,6 +61,10 @@ pub const MAX_SIDE: f32 = 8192.0; /// lines. const THUMB_TEXT_BYTES: usize = 2048; +/// A page shown in full is rendered at most this many pixels on its +/// longest side: a letter page at twice its points (612 x 792). +const PAGE_PX: u32 = 1584; + /// Where the live viewer keeps thumbnails: the user's cache directory /// (`$XDG_CACHE_HOME`, else `~/.cache`), `wanderbooks/thumbs/v1-560`; /// the version and width name how they were drawn, so a change in either @@ -322,23 +326,27 @@ fn render_thumb(job: &Job, dir: Option<&std::path::Path>) -> Result THUMB_PX { - let h = (img.height() as f32 * THUMB_PX / img.width() as f32).max(1.0); - img.resize_exact( - THUMB_PX as u32, - h as u32, - image::imageops::FilterType::Triangle, - ) - } else { - img - } + image::load_from_memory(&bytes).map_err(|e| e.to_string())? } else { return Err(format!("{} can't be previewed", job.mime)); }; + let img = if img.width() as f32 > THUMB_PX { + let h = (img.height() as f32 * THUMB_PX / img.width() as f32).max(1.0); + img.resize_exact( + THUMB_PX as u32, + h as u32, + image::imageops::FilterType::Triangle, + ) + } else { + img + }; let img = img.crop_imm(0, 0, img.width(), img.height().min(THUMB_MAX_H)); if let Some(path) = &kept { let mut png = Vec::new(); @@ -385,43 +393,13 @@ fn keep(path: &std::path::Path, png: &[u8]) { } } -/// A PDF's first page as a PNG, [`THUMB_PX`] wide. -fn pdf_first_page_png(bytes: Vec) -> Result, String> { - use hayro::hayro_interpret::InterpreterSettings; - use hayro::hayro_syntax::Pdf; - use hayro::vello_cpu::color::palette::css::WHITE; - use hayro::{PixmapSettings, RenderCache, RenderSettings, render}; - let pdf = Pdf::new(bytes).map_err(|e| format!("not a readable PDF: {e:?}"))?; - let pages = pdf.pages(); - let page = pages.first().ok_or("the PDF has no pages")?; - let (w, h) = page.render_dimensions(); - // THUMB_PX wide, but never past a texture's side on a long page (the - // picture is cut to THUMB_MAX_H after), and never nothing. - let scale = (THUMB_PX / w.max(1.0)) - .min(MAX_SIDE / h.max(1.0)) - .max(1.0 / w.max(h).max(1.0)); - let px = PixmapSettings { - x_scale: scale, - y_scale: scale, - bg_color: WHITE, - }; - let pixmap = render( - page, - &RenderCache::new(), - &InterpreterSettings::default(), - &RenderSettings::default(), - &px, - ); - pixmap.into_png().map_err(|e| format!("{e:?}")) -} - fn render(job: &Job) -> Result { let bytes = BlobStore::open_read_only(&job.root) .get(&job.sha) .map_err(|e| e.to_string())?; let is_pdf = job.mime == "application/pdf" || bytes.starts_with(b"%PDF"); if is_pdf { - render_pdf(bytes).map(Rendered::Pages) + render_pdf(&bytes).map(Rendered::Pages) } else if job.mime.starts_with("image/") { let img = image::load_from_memory(&bytes).map_err(|e| e.to_string())?; // Past a texture's side, scaled down to fit (aspect kept). @@ -478,27 +456,15 @@ pub fn text_of(bytes: Vec) -> Result { Ok(text) } -fn render_pdf(bytes: Vec) -> Result, String> { - use hayro::hayro_interpret::InterpreterSettings; - use hayro::hayro_syntax::Pdf; - use hayro::vello_cpu::color::palette::css::WHITE; - use hayro::{PixmapSettings, RenderCache, RenderSettings, render}; - let pdf = Pdf::new(bytes).map_err(|e| format!("not a readable PDF: {e:?}"))?; - let cache = RenderCache::new(); - let interp = InterpreterSettings::default(); - let settings = RenderSettings::default(); +/// A PDF's pages, in order, through the one extractor (`ledger-import`, +/// plan P6): each at most [`PAGE_PX`] pixels on its longest side, at most +/// [`ledger_import::extract::MAX_PAGES`] pages. +fn render_pdf(bytes: &[u8]) -> Result, String> { + let pages = ledger_import::extract::page_count(bytes).map_err(|e| e.to_string())?; let mut out = Vec::new(); - for page in pdf.pages().iter().take(30) { - // Twice the page's size, but never past a texture's side. - let (w, h) = page.render_dimensions(); - let scale = 2.0f32.min(MAX_SIDE / w.max(h).max(1.0)); - let px = PixmapSettings { - x_scale: scale, - y_scale: scale, - bg_color: WHITE, - }; - let pixmap = render(page, &cache, &interp, &settings, &px); - let png = pixmap.into_png().map_err(|e| format!("{e:?}"))?; + for page in 0..pages.min(ledger_import::extract::MAX_PAGES) { + let png = + ledger_import::extract::page_png(bytes, page, PAGE_PX).map_err(|e| e.to_string())?; let img = image::load_from_memory(&png).map_err(|e| e.to_string())?; out.push(to_color_image(img.to_rgba8())); } diff --git a/crates/ledger-host/src/commands.rs b/crates/ledger-host/src/commands.rs index 4c709d0..08f8a54 100644 --- a/crates/ledger-host/src/commands.rs +++ b/crates/ledger-host/src/commands.rs @@ -477,6 +477,7 @@ fn books_usage(prog: &str) { eprintln!(" {prog} restore BACKUP [--id ID] [--replace] [--yes]"); eprintln!(" {prog} back-up ID [--new-disk]"); eprintln!(" {prog} upgrade ID"); + eprintln!(" {prog} extract BOOK ATTACHMENT"); eprintln!( " {prog} put-aside ID DATA-DIR [--vault DIR --vault-key FILE] [PATH...] [--yes]" ); @@ -492,6 +493,9 @@ fn books_usage(prog: &str) { eprintln!( " upgrade give the go-ahead, with the owner passphrase, for books an older wanderbooks wrote to move to this version's journal format (a verified backup with the vault and keys first)" ); + eprintln!( + " extract read one attachment's text (a PDF, XLSX or text file) and journal it, one at a time; books an older version filed are read this way after an upgrade" + ); eprintln!( " put-aside move the old copies of adopted books (their data folder, their vault and its key, and other paths given: settings file, git history) whole into the legacy folder, once the books hold everything they do, byte for byte; nothing is deleted" ); @@ -580,6 +584,19 @@ pub fn books( } ["restore", source] => return restore(&common, source, id, replace, yes), ["upgrade", book] => return upgrade(&common, book), + ["extract", book, attachment] => { + let attachment_id: u64 = attachment.parse().map_err(|_| { + eprintln!("error: extract takes an attachment id: {attachment}"); + exit(2) + })?; + with_server( + &common, + Request::Extract { + id: (*book).to_string(), + attachment_id, + }, + ) + } ["put-aside", book, source, also @ ..] => { let also: Vec = also.iter().map(PathBuf::from).collect(); let old = ledger_store::root::OldCopies { @@ -634,6 +651,25 @@ pub fn books( ); Err(exit(1)) } + Ok(Response::Extracted { + attachment_id, + status, + pages, + text_sha256, + }) => { + match (&text_sha256, status.as_str()) { + (Some(sha), "ok") => { + println!("Read attachment {attachment_id}: {pages} page(s), text {sha}.") + } + (_, "none") => println!( + "Read attachment {attachment_id}: no text to read ({pages} page(s)); view its pages as pictures." + ), + _ => println!( + "Read attachment {attachment_id}: the reader failed ({pages} page(s)); a human must read it." + ), + } + Ok(()) + } Ok(Response::Refused(why)) | Err(why) => { eprintln!("error: {why}"); Err(exit(1)) diff --git a/crates/ledger-import/Cargo.toml b/crates/ledger-import/Cargo.toml index 5900683..a6e690f 100644 --- a/crates/ledger-import/Cargo.toml +++ b/crates/ledger-import/Cargo.toml @@ -18,6 +18,17 @@ typst = "0.15.1" serde_json = "1" typst-layout = "0.15.1" qrcode = { version = "0.14.1", default-features = false, features = ["svg"] } +# Extraction (extract.rs, plan P6): a PDF's text, a page's pixels, an +# XLSX's cells. hayro (the renderer) and hayro-interpret (the text +# interpreter) are pure Rust and already the viewer's PDF reader, so the +# server and the viewer read a receipt through one implementation rather +# than two; `zip` and `quick-xml` read a spreadsheet's XML. None of them +# touches money: a number is kept as the decimal string the XML holds. +hayro = { version = "0.8", features = ["embed-fonts"] } +hayro-interpret = "0.8" +kurbo = "0.13" +zip = { version = "9.0.1", default-features = false, features = ["deflate"] } +quick-xml = "0.38" # BSD-3-Clause: the IIF's transliteration to ASCII (iif.rs, rules.md B-34). # Pinned exactly: a new table would change what a CPA's file says. deunicode = { version = "=1.6.0", default-features = false } @@ -27,8 +38,4 @@ workspace = true [dev-dependencies] proptest = "1" -# The viewer's PDF reader (ledger-gui's hayro): reads back the text a PDF -# draws, glyph by glyph, in the tests. -hayro-interpret = "0.8" -kurbo = "0.13" rqrr = { version = "0.11.0", default-features = false } diff --git a/crates/ledger-import/src/extract.rs b/crates/ledger-import/src/extract.rs new file mode 100644 index 0000000..0a03f63 --- /dev/null +++ b/crates/ledger-import/src/extract.rs @@ -0,0 +1,625 @@ +//! Reading the words and pages out of the files the books keep: a PDF's +//! text and a page as a picture, an XLSX's cells as exact decimal strings, +//! and plain text as it is (plan P6 step 1, D-32, D-53). +//! +//! ```text +//! GOAL: one extractor, in this process, that the server's filing scan, +//! the journal's extraction record and the viewer all read a file +//! through, so a receipt's text never depends on which door it came +//! in; a reader failure is an error, never a panic out of the +//! server. +//! SOURCE: plan/served.md P6 step 1 (D-32: no sandbox; D-53: in-process, +//! no new crate); IRS Pub. 583 (records must be readable); D-15 +//! (no OCR: an image-only receipt is its page pictures). +//! INVARIANTS: +//! 1. Deterministic: the same bytes give the same text and the same PNG +//! bytes, every run. +//! 2. Text is read, never guessed: an XLSX cell's number is the decimal +//! string the sheet XML holds, never a binary float; a PDF's text is +//! its glyphs' ToUnicode text, page by page, line by line. +//! 3. Caps: at most 25 MiB of input, 30 pages, 2 MiB of text. Past the +//! input cap nothing is read; past the page cap the rest is not +//! read; past the text cap the text is cut at a character boundary. +//! 4. A panic inside the PDF reader is caught here and reported as +//! [`ExtractError::ReaderPanicked`]: the process is not isolated +//! (D-32, and the plan says so). +//! ``` + +use std::io::{Cursor, Read}; +use thiserror::Error; + +/// The largest file extraction reads: 25 MiB (plan P6 step 1). +pub const MAX_BYTES: usize = 25 * 1024 * 1024; + +/// The most pages of a PDF extraction reads. +pub const MAX_PAGES: usize = 30; + +/// The most text one extraction keeps: 2 MiB. +pub const MAX_TEXT_BYTES: usize = 2 * 1024 * 1024; + +/// What read a file's text, as the extraction record names it. +pub const EXTRACTOR: &str = "wanderbooks"; + +/// That extractor's version: a new one journals a new extraction record, +/// never replaces an old one (plan P6 step 2). +pub const VERSION: &str = env!("CARGO_PKG_VERSION"); + +/// What reading a file found (plan P6 step 2's `status`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ExtractStatus { + /// Text was read. + Ok, + /// There is no text to read (an image is its page pictures, D-15). + None, + /// Reading was tried and failed. + Failed, +} + +impl ExtractStatus { + /// Its wire name, as the extraction record journals it. + pub fn key(self) -> &'static str { + match self { + ExtractStatus::Ok => "ok", + ExtractStatus::None => "none", + ExtractStatus::Failed => "failed", + } + } +} + +/// What [`extract`] found: the text (when any), the file's page count, and +/// whether there was text to read. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Extracted { + /// The text, capped at [`MAX_TEXT_BYTES`]. + pub text: Option, + /// How many pages the file has (0 when it has none). + pub pages: u32, + /// What was found. + pub status: ExtractStatus, +} + +/// Why a file's text or page could not be read. +#[derive(Debug, Error)] +pub enum ExtractError { + /// The file is past [`MAX_BYTES`]. + #[error("the file is {size} bytes: extraction reads at most {MAX_BYTES} bytes")] + TooLarge { + /// The file's size. + size: usize, + }, + /// The bytes are not a PDF this reader can read. + #[error("not a readable PDF: {0}")] + Pdf(String), + /// The bytes are not a spreadsheet this reader can read. + #[error("not a readable spreadsheet: {0}")] + Xlsx(String), + /// The page asked for is past the document's last. + #[error("page {page} is past the document's {pages} page(s)")] + PageOutOfRange { + /// The page asked for (1-based). + page: usize, + /// How many pages the document has. + pages: usize, + }, + /// The page size asked for is zero. + #[error("a page image needs a size of at least 1 pixel")] + NoPixels, + /// The PDF reader panicked on the file (INVARIANT 4). + #[error("the PDF reader stopped on that file")] + ReaderPanicked, +} + +/// Run `read`, turning a panic into [`ExtractError::ReaderPanicked`] +/// (INVARIANT 4). +fn caught(read: impl FnOnce() -> Result) -> Result { + std::panic::catch_unwind(std::panic::AssertUnwindSafe(read)) + .unwrap_or(Err(ExtractError::ReaderPanicked)) +} + +/// The text a PDF draws: each glyph's text, left to right, one `String` +/// per glyph, one `Vec` per line, from the top of each page down, page +/// after page (INVARIANT 2). At most [`MAX_PAGES`] pages are read. +pub fn pdf_lines(bytes: &[u8]) -> Result>, ExtractError> { + if bytes.len() > MAX_BYTES { + return Err(ExtractError::TooLarge { size: bytes.len() }); + } + caught(|| read_pdf_lines(bytes)) +} + +/// The text a PDF draws, its glyphs joined into lines and its pages into +/// one text (INVARIANTS 1 and 2), capped at [`MAX_TEXT_BYTES`]. +pub fn pdf_text(bytes: &[u8]) -> Result { + let lines = pdf_lines(bytes)?; + let text = lines + .iter() + .map(|l| l.concat()) + .collect::>() + .join("\n"); + Ok(cap_text(text)) +} + +/// How many pages a PDF has. +pub fn page_count(bytes: &[u8]) -> Result { + if bytes.len() > MAX_BYTES { + return Err(ExtractError::TooLarge { size: bytes.len() }); + } + caught(|| Ok(read_pdf(bytes)?.pages().len())) +} + +/// One page of a PDF as a PNG whose longest side is at most `max_side_px` +/// pixels (`page` is 0-based; `max_side_px` of 0 is refused). The same +/// bytes, page and size give the same PNG bytes (INVARIANT 1). +pub fn page_png(bytes: &[u8], page: usize, max_side_px: u32) -> Result, ExtractError> { + if bytes.len() > MAX_BYTES { + return Err(ExtractError::TooLarge { size: bytes.len() }); + } + if max_side_px == 0 { + return Err(ExtractError::NoPixels); + } + caught(|| read_page_png(bytes, page, max_side_px)) +} + +/// The bytes as text, if they are text the books read (INVARIANT 2): a +/// PDF's text, an XLSX's cells, UTF-16 with its byte-order mark, UTF-8, or, +/// for a text type or name, whatever they decode to. Text with NUL bytes +/// and no byte-order mark (UTF-16 written without one is valid UTF-8, its +/// digits split by NULs) is read every way it can be (as it is, as UTF-16 +/// both ways round, and with its NULs taken out), each reading on a line of +/// its own: so is bytes that aren't UTF-8 but are NUL-interleaved as UTF-16 +/// is. At most [`MAX_TEXT_BYTES`] is returned. +pub fn text_of(bytes: &[u8], mime_type: &str, file_name: &str) -> Option { + extract(bytes, mime_type, file_name).text +} + +/// Read a file the way the books record it (plan P6 step 2): a PDF's text +/// and pages, an XLSX's cells, plain text as it is; an image has none. A +/// reader failure is [`ExtractStatus::Failed`], never a panic (INVARIANT 4). +pub fn extract(bytes: &[u8], mime_type: &str, file_name: &str) -> Extracted { + if bytes.len() > MAX_BYTES { + return Extracted { + text: None, + pages: 0, + status: ExtractStatus::Failed, + }; + } + let mime = mime_type.trim().to_ascii_lowercase(); + let ext = std::path::Path::new(file_name) + .extension() + .and_then(|e| e.to_str()) + .map(str::to_ascii_lowercase) + .unwrap_or_default(); + let magic_pdf = bytes.starts_with(b"%PDF"); + if mime == "application/pdf" || magic_pdf { + return match pdf_text(bytes) { + Ok(text) => { + let pages = page_count(bytes).unwrap_or(0).min(MAX_PAGES) as u32; + found(Some(text), pages) + } + // A file CALLED a PDF whose bytes are text (a text file named + // .pdf): read the text, so the TIN scan and the reader see what + // it says (B-26: the scan fails closed). One with the PDF magic + // that won't read is a failed extraction. + Err(_) if !magic_pdf => match plain_text(bytes, &mime, &ext) { + Some(text) => found(Some(text), 1), + None => Extracted { + text: None, + pages: 0, + status: ExtractStatus::Failed, + }, + }, + Err(_) => Extracted { + text: None, + pages: 0, + status: ExtractStatus::Failed, + }, + }; + } + let xlsx = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + if mime == xlsx || ext == "xlsx" || ext == "xlsm" { + return match xlsx_text(bytes) { + Ok(text) => found(Some(text), 1), + Err(_) => Extracted { + text: None, + pages: 0, + status: ExtractStatus::Failed, + }, + }; + } + match plain_text(bytes, &mime, &ext) { + Some(text) => found(Some(text), 1), + None => Extracted { + text: None, + pages: 0, + status: ExtractStatus::None, + }, + } +} + +/// An extraction's answer from the text read: `none` when it is blank. +fn found(text: Option, pages: u32) -> Extracted { + match text { + Some(t) if t.trim().is_empty() => Extracted { + text: None, + pages, + status: ExtractStatus::None, + }, + Some(t) => Extracted { + text: Some(t), + pages, + status: ExtractStatus::Ok, + }, + None => Extracted { + text: None, + pages, + status: ExtractStatus::None, + }, + } +} + +/// The plain-text reading of bytes (the second half of [`text_of`]). +fn plain_text(bytes: &[u8], mime: &str, ext: &str) -> Option { + let utf16 = |body: &[u8], le: bool| { + let units: Vec = body + .as_chunks::<2>() + .0 + .iter() + .map(|&c| { + if le { + u16::from_le_bytes(c) + } else { + u16::from_be_bytes(c) + } + }) + .collect(); + String::from_utf16_lossy(&units) + }; + let text = match bytes { + [0xFF, 0xFE, body @ ..] => utf16(body, true), + [0xFE, 0xFF, body @ ..] => utf16(body, false), + _ => { + let plain = match std::str::from_utf8(bytes) { + Ok(t) => Some(t.to_string()), + Err(_) if text_like(mime, ext) => Some(String::from_utf8_lossy(bytes).into_owned()), + Err(_) => None, + }; + if !bytes.contains(&0) || plain.is_none() && !nul_interleaved(bytes) { + return plain.map(cap_text); + } + let without_nuls: Vec = bytes.iter().copied().filter(|&b| b != 0).collect(); + let readings = [ + plain.unwrap_or_default(), + utf16(bytes, true), + utf16(bytes, false), + String::from_utf8_lossy(&without_nuls).into_owned(), + ]; + readings.join("\n") + } + }; + Some(cap_text(text)) +} + +/// [`text_of`]'s answer capped at [`MAX_TEXT_BYTES`], cut at a character +/// boundary. +fn cap_text(mut text: String) -> String { + if text.len() > MAX_TEXT_BYTES { + let mut cut = MAX_TEXT_BYTES; + while !text.is_char_boundary(cut) { + cut -= 1; + } + text.truncate(cut); + } + text +} + +/// Whether a file of this type or name is text: csv, tsv, txt, json, iif, +/// xml, html, md and their kin. +fn text_like(mime: &str, ext: &str) -> bool { + mime.starts_with("text/") + || [ + "json", + "xml", + "csv", + "x-iif", + "javascript", + "x-yaml", + "yaml", + ] + .iter() + .any(|t| mime.ends_with(&format!("/{t}")) || mime.ends_with(&format!("+{t}"))) + || [ + "csv", "tsv", "txt", "text", "json", "iif", "xml", "html", "htm", "md", "markdown", + "yaml", "yml", "tab", "prn", "log", "ofx", "qfx", "qif", + ] + .contains(&ext) +} + +/// Whether `bytes` look like UTF-16 without a byte-order mark: of their +/// byte pairs, at least half have a NUL on one side (the high byte of a +/// Latin character) and at most an eighth on the other. +#[doc(hidden)] +pub fn nul_interleaved(bytes: &[u8]) -> bool { + let pairs = bytes.as_chunks::<2>().0; + let nuls = |side: usize| pairs.iter().filter(|p| p[side] == 0).count(); + let (first, second) = (nuls(0), nuls(1)); + let n = pairs.len(); + n > 0 && (2 * first >= n && 8 * second <= n || 2 * second >= n && 8 * first <= n) +} + +// --------------------------------------------------------------------------- +// PDF +// --------------------------------------------------------------------------- + +type Pdf = hayro_interpret::hayro_syntax::Pdf; + +/// [`Pdf::new`] with its error as ours. +fn read_pdf(bytes: &[u8]) -> Result { + Pdf::new(bytes.to_vec()).map_err(|e| ExtractError::Pdf(format!("{e:?}"))) +} + +fn read_pdf_lines(bytes: &[u8]) -> Result>, ExtractError> { + use hayro_interpret::font::GlyphRun; + use hayro_interpret::hayro_cmap::BfString; + use hayro_interpret::{ + BlendMode, ClipPath, Context, Device, DrawMode, DrawProps, Image, ImageDrawProps, + InterpreterCache, InterpreterSettings, SoftMask, interpret_page, + }; + use kurbo::{Affine, BezPath, Point, Rect}; + /// Each glyph drawn: its baseline (hundredths of a point), x, and text. + struct Glyphs(Vec<(i64, f64, String)>); + impl<'a> Device<'a> for Glyphs { + fn draw_path(&mut self, _: &BezPath, _: DrawProps<'a>, _: &DrawMode) {} + fn push_clip_path(&mut self, _: &ClipPath) {} + fn push_transparency_group(&mut self, _: f32, _: Option>, _: BlendMode) {} + fn draw_glyph_run(&mut self, run: &GlyphRun<'_, 'a>, props: DrawProps<'a>, _: &DrawMode) { + for g in run.glyphs() { + let at = (props.transform * g.transform()) * Point::ZERO; + let text = match g.as_unicode() { + Some(BfString::Char(c)) => c.to_string(), + Some(BfString::String(s)) => s, + // A mark or a ligature's second glyph: its text is with + // the cluster's first. + None => String::new(), + }; + self.0.push(((at.y * 100.0).round() as i64, at.x, text)); + } + } + fn draw_image(&mut self, _: Image<'a, '_>, _: ImageDrawProps<'a>) {} + fn pop_clip(&mut self) {} + fn pop_transparency_group(&mut self) {} + } + let pdf = read_pdf(bytes)?; + let cache = InterpreterCache::new(); + let mut out = Vec::new(); + for page in pdf.pages().iter().take(MAX_PAGES) { + let mut ctx = Context::new( + Affine::IDENTITY, + Rect::new(0.0, 0.0, 1.0, 1.0), + &cache, + pdf.xref(), + InterpreterSettings::default(), + ); + let mut glyphs = Glyphs(Vec::new()); + interpret_page(page, &mut ctx, &mut glyphs); + // PDF space runs up the page: the highest baseline first. + let mut lines: std::collections::BTreeMap, Vec<(f64, String)>> = + std::collections::BTreeMap::new(); + for (y, x, t) in glyphs.0 { + lines.entry(std::cmp::Reverse(y)).or_default().push((x, t)); + } + for (_, mut line) in lines { + line.sort_by(|a, b| a.0.total_cmp(&b.0)); + out.push(line.into_iter().map(|(_, t)| t).collect()); + } + } + Ok(out) +} + +fn read_page_png(bytes: &[u8], page: usize, max_side_px: u32) -> Result, ExtractError> { + use hayro::hayro_interpret::InterpreterSettings; + use hayro::vello_cpu::color::palette::css::WHITE; + use hayro::{PixmapSettings, RenderCache, RenderSettings, render}; + let pdf = read_pdf(bytes)?; + let pages = pdf.pages(); + let found = pages.get(page).ok_or(ExtractError::PageOutOfRange { + page: page + 1, + pages: pages.len(), + })?; + let (w, h) = found.render_dimensions(); + // The page's longest side at most `max_side_px`; never zero. + let scale = max_side_px as f32 / w.max(h).max(1.0); + let scale = scale.max(1.0 / w.max(h).max(1.0)); + let px = PixmapSettings { + x_scale: scale, + y_scale: scale, + bg_color: WHITE, + }; + let pixmap = render( + found, + &RenderCache::new(), + &InterpreterSettings::default(), + &RenderSettings::default(), + &px, + ); + pixmap + .into_png() + .map_err(|e| ExtractError::Pdf(format!("{e:?}"))) +} + +// --------------------------------------------------------------------------- +// XLSX +// --------------------------------------------------------------------------- + +/// An XLSX's cells as text: one line per row, one cell per column, +/// separated by tabs; a number is the exact decimal string the sheet XML +/// holds, never a binary float (INVARIANT 2). +pub fn xlsx_text(bytes: &[u8]) -> Result { + if bytes.len() > MAX_BYTES { + return Err(ExtractError::TooLarge { size: bytes.len() }); + } + let mut zip = + zip::ZipArchive::new(Cursor::new(bytes)).map_err(|e| ExtractError::Xlsx(e.to_string()))?; + // The workbook's shared strings, in index order. + let shared = match read_zip(&mut zip, "xl/sharedStrings.xml") { + Some(xml) => shared_strings(&xml)?, + None => Vec::new(), + }; + let names: Vec = zip + .file_names() + .filter_map(|n| n.ok().map(|n| n.into_owned())) + .filter(|n| n.starts_with("xl/worksheets/sheet") && n.ends_with(".xml")) + .collect(); + let mut sheets = names; + sheets.sort(); + let mut out = String::new(); + for name in sheets { + let Some(xml) = read_zip(&mut zip, &name) else { + continue; + }; + let text = sheet_cells(&xml, &shared)?; + if !text.is_empty() { + if !out.is_empty() { + out.push('\n'); + } + out.push_str(&text); + } + } + Ok(cap_text(out)) +} + +/// A zip member's text, if it is there. +fn read_zip(zip: &mut zip::ZipArchive>, name: &str) -> Option { + let mut file = zip.by_name(name).ok()?; + let mut bytes = Vec::new(); + file.read_to_end(&mut bytes).ok()?; + Some(String::from_utf8_lossy(&bytes).into_owned()) +} + +/// The text of each `` in `xl/sharedStrings.xml`, rich-text runs +/// concatenated. +fn shared_strings(xml: &str) -> Result, ExtractError> { + let mut reader = quick_xml::Reader::from_str(xml); + let mut out = Vec::new(); + let mut current: Option = None; + let mut in_text = false; + loop { + match reader.read_event() { + Ok(quick_xml::events::Event::Start(e)) if e.name().as_ref() == b"si" => { + current = Some(String::new()); + } + Ok(quick_xml::events::Event::End(e)) if e.name().as_ref() == b"si" => { + out.push(current.take().unwrap_or_default()); + } + Ok(quick_xml::events::Event::Start(e)) if e.name().as_ref() == b"t" => in_text = true, + Ok(quick_xml::events::Event::End(e)) if e.name().as_ref() == b"t" => in_text = false, + Ok(quick_xml::events::Event::Text(t)) if in_text => { + if let Some(s) = &mut current { + s.push_str(&t.decode().map_err(|e| ExtractError::Xlsx(e.to_string()))?); + } + } + Ok(quick_xml::events::Event::Eof) => break, + Ok(_) => {} + Err(e) => return Err(ExtractError::Xlsx(e.to_string())), + } + } + Ok(out) +} + +/// One worksheet's cells as tab-separated rows. +fn sheet_cells(xml: &str, shared: &[String]) -> Result { + use quick_xml::events::Event; + let mut reader = quick_xml::Reader::from_str(xml); + let mut rows: Vec> = Vec::new(); + let mut row: Vec = Vec::new(); + let mut cell_type = String::new(); + let mut in_value = false; + let mut value = String::new(); + let mut in_inline = false; + let mut inline = String::new(); + let mut in_inline_text = false; + loop { + match reader.read_event() { + Ok(Event::Start(e)) => match e.name().as_ref() { + b"row" => row = Vec::new(), + b"c" => { + cell_type.clear(); + value.clear(); + inline.clear(); + for a in e.attributes().flatten() { + if a.key.as_ref() == b"t" { + cell_type = String::from_utf8_lossy(&a.value).into_owned(); + } + } + } + b"v" => { + in_value = true; + value.clear(); + } + b"is" => in_inline = true, + b"t" if in_inline => in_inline_text = true, + _ => {} + }, + Ok(Event::End(e)) => match e.name().as_ref() { + b"row" => rows.push(std::mem::take(&mut row)), + b"c" => { + let text = match cell_type.as_str() { + "s" => value + .parse::() + .ok() + .and_then(|i| shared.get(i).cloned()) + .unwrap_or_default(), + "inlineStr" => inline.clone(), + "b" => match value.as_str() { + "1" => "TRUE".into(), + _ => "FALSE".into(), + }, + _ => value.clone(), + }; + row.push(text); + } + b"v" => in_value = false, + b"is" => in_inline = false, + b"t" if in_inline => in_inline_text = false, + _ => {} + }, + Ok(Event::Text(t)) => { + let text = t.decode().map_err(|e| ExtractError::Xlsx(e.to_string()))?; + if in_inline_text { + inline.push_str(&text); + } else if in_value { + value.push_str(&text); + } + } + Ok(Event::Eof) => break, + Ok(_) => {} + Err(e) => return Err(ExtractError::Xlsx(e.to_string())), + } + } + Ok(rows + .iter() + .map(|r| r.join("\t")) + .collect::>() + .join("\n")) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// INVARIANT 4: a panic inside the reader is caught, never fatal. + #[test] + fn a_reader_panic_is_reported_not_raised() { + let got = caught::<()>(|| panic!("the reader broke")); + assert!(matches!(got, Err(ExtractError::ReaderPanicked))); + } + + /// INVARIANT 3: a file past the input cap is not read. + #[test] + fn too_large_is_refused_without_reading() { + let bytes = vec![0u8; MAX_BYTES + 1]; + assert!(matches!( + pdf_text(&bytes), + Err(ExtractError::TooLarge { .. }) + )); + assert!(text_of(&bytes, "text/plain", "big.txt").is_none()); + } +} diff --git a/crates/ledger-import/src/lib.rs b/crates/ledger-import/src/lib.rs index 53cb709..4d479d1 100644 --- a/crates/ledger-import/src/lib.rs +++ b/crates/ledger-import/src/lib.rs @@ -16,6 +16,7 @@ pub mod asset_register; pub mod bank_csv; pub mod chart_csv; +pub mod extract; pub mod fonts; pub mod iif; pub mod invoice_pdf; @@ -28,6 +29,7 @@ pub mod xlsx; pub use asset_register::{asset_register_pdf, asset_register_xlsx}; pub use bank_csv::{BankCsvError, BankTransaction, DateOrder, parse_bank_csv, parse_bank_csv_with}; pub use chart_csv::{ChartRow, parse_chart_csv}; +pub use extract::{ExtractError, page_count, page_png, pdf_lines, pdf_text, text_of, xlsx_text}; pub use iif::{ Iif, IifAccount, IifChange, IifClass, IifError, IifField, LeftOut, MEMO_MAX, NAME_MAX, NUMBER_MAX, QbAccountType, iif_accounts, iif_classes, journal_to_iif, diff --git a/crates/ledger-import/tests/extract.rs b/crates/ledger-import/tests/extract.rs new file mode 100644 index 0000000..20cbee6 --- /dev/null +++ b/crates/ledger-import/tests/extract.rs @@ -0,0 +1,126 @@ +//! The one extractor (plan P6 step 1): a PDF's text and pages, and an +//! XLSX's cells as exact decimal strings. +//! +//! Expected behavior comes from `crates/ledger-import/src/extract.rs`'s +//! INVARIANTS, which come from plan/served.md P6 step 1 and D-15, D-32 and +//! D-53. + +use std::io::{Cursor, Write}; + +use ledger_import::extract::{ExtractError, MAX_BYTES, page_png, pdf_text, text_of, xlsx_text}; +use ledger_import::report::{ReportMeta, Row, RowKind, statement}; +use ledger_import::statement_pdf; +use proptest::prelude::*; +use rust_decimal::Decimal; + +fn a_pdf() -> Vec { + let st = statement( + "Receipt", + "March 2026".to_string(), + &["Total"], + vec![Row { + kind: RowKind::Line, + label: "Rent".into(), + amounts: vec![Some(Decimal::new(10000, 2))], + currency: true, + rule_above: false, + }], + ); + let meta = ReportMeta { + company: "Fernhill Labs LLC".into(), + basis: "Accrual".into(), + prepared_on: chrono::NaiveDate::from_ymd_opt(2026, 3, 2).unwrap(), + journal_seq: 1, + }; + statement_pdf(&st, &meta).unwrap() +} + +/// An XLSX holding one sheet whose cells are `values` (numbers as written, +/// not through a float). +fn xlsx_of(values: &[String]) -> Vec { + let mut writer = zip::ZipWriter::new(Cursor::new(Vec::new())); + let options = zip::write::SimpleFileOptions::default(); + writer + .start_file("xl/worksheets/sheet1.xml", options) + .unwrap(); + let cells: String = values + .iter() + .enumerate() + .map(|(i, v)| format!(r#"{v}"#, i + 1)) + .collect(); + writer + .write_all( + format!( + r#" +{cells}"# + ) + .as_bytes(), + ) + .unwrap(); + writer.finish().unwrap().into_inner() +} + +/// INVARIANT 1: the same bytes give the same text and the same PNG bytes, +/// every run. +#[test] +fn extraction_deterministic() { + let pdf = a_pdf(); + let first = pdf_text(&pdf).unwrap(); + assert_eq!(first, pdf_text(&pdf).unwrap(), "the text"); + assert!(first.contains("Rent"), "{first}"); + let first_png = page_png(&pdf, 0, 600).unwrap(); + assert_eq!(first_png, page_png(&pdf, 0, 600).unwrap(), "the page"); + assert!(first_png.starts_with(b"\x89PNG\r\n\x1a\n"), "a PNG"); + // A plain text file reads back exactly; an image has no text (D-15). + assert_eq!( + text_of(b"date,amount\n2026-03-01,12.50\n", "text/csv", "bank.csv").as_deref(), + Some("date,amount\n2026-03-01,12.50\n") + ); + assert_eq!(text_of(b"\x89PNG\r\n\x1a\n", "image/png", "r.png"), None); +} + +// INVARIANT 2: an XLSX cell's number is the exact decimal string the +// sheet XML holds, never a binary float. +proptest! { + #[test] + fn xlsx_cells_are_decimal_strings(values in proptest::collection::vec( + (-1_000_000_000_000i64..1_000_000_000_000i64, 0u32..=6u32), 1..12)) { + let cells: Vec = values + .iter() + .map(|(mantissa, scale)| Decimal::new(*mantissa, *scale).to_string()) + .collect(); + let text = xlsx_text(&xlsx_of(&cells)).unwrap(); + let got: Vec<&str> = text.split('\t').collect(); + prop_assert_eq!(got, cells.iter().map(String::as_str).collect::>()); + } +} + +/// INVARIANT 4 and D-32: a file the reader can't read is an error that +/// says so, not a panic out of the server. +#[test] +fn a_pdf_that_crashes_the_reader_is_failed_not_fatal() { + for bytes in [ + &b"not a pdf at all"[..], + &b"%PDF-1.7\n1 0 obj << /Type /Catalog"[..], + &b""[..], + ] { + assert!( + matches!(pdf_text(bytes), Err(ExtractError::Pdf(_))), + "{bytes:?}" + ); + } + // The input cap is a refusal too, before the reader runs. + let too_big = vec![0u8; MAX_BYTES + 1]; + assert!(matches!( + pdf_text(&too_big), + Err(ExtractError::TooLarge { .. }) + )); + assert_eq!(text_of(&too_big, "application/pdf", "x.pdf"), None); + // No page past the last, and no zero-sized page. + let pdf = a_pdf(); + assert!(matches!( + page_png(&pdf, 30, 600), + Err(ExtractError::PageOutOfRange { .. }) + )); + assert!(matches!(page_png(&pdf, 0, 0), Err(ExtractError::NoPixels))); +} diff --git a/crates/ledger-import/tests/spec_import.rs b/crates/ledger-import/tests/spec_import.rs index 6d28643..8c0db08 100644 --- a/crates/ledger-import/tests/spec_import.rs +++ b/crates/ledger-import/tests/spec_import.rs @@ -3347,64 +3347,10 @@ fn pdf_keeps_chinese_japanese_and_korean_names() { /// The text a PDF draws, each line from the top of each page down, its /// glyphs left to right, each the text its font's ToUnicode map gives -/// back, read with hayro (the viewer's own PDF reader): what a reader -/// copies, searches and speaks. +/// back: the one extractor (`ledger_import::extract`, plan P6), which the +/// server and the viewer read receipts through too. fn pdf_lines(pdf: &[u8]) -> Vec> { - use hayro_interpret::font::GlyphRun; - use hayro_interpret::hayro_cmap::BfString; - use hayro_interpret::hayro_syntax::Pdf; - use hayro_interpret::{ - BlendMode, ClipPath, Context, Device, DrawMode, DrawProps, Image, ImageDrawProps, - InterpreterCache, InterpreterSettings, SoftMask, interpret_page, - }; - use kurbo::{Affine, BezPath, Point, Rect}; - /// Each glyph drawn: its baseline (tenths of a point), x, and text. - struct Glyphs(Vec<(i64, f64, String)>); - impl<'a> Device<'a> for Glyphs { - fn draw_path(&mut self, _: &BezPath, _: DrawProps<'a>, _: &DrawMode) {} - fn push_clip_path(&mut self, _: &ClipPath) {} - fn push_transparency_group(&mut self, _: f32, _: Option>, _: BlendMode) {} - fn draw_glyph_run(&mut self, run: &GlyphRun<'_, 'a>, props: DrawProps<'a>, _: &DrawMode) { - for g in run.glyphs() { - let at = (props.transform * g.transform()) * Point::ZERO; - let text = match g.as_unicode() { - Some(BfString::Char(c)) => c.to_string(), - Some(BfString::String(s)) => s, - // A mark or a ligature's second glyph: its text is - // with the cluster's first. - None => String::new(), - }; - self.0.push(((at.y * 10.0).round() as i64, at.x, text)); - } - } - fn draw_image(&mut self, _: Image<'a, '_>, _: ImageDrawProps<'a>) {} - fn pop_clip(&mut self) {} - fn pop_transparency_group(&mut self) {} - } - let pdf = Pdf::new(pdf.to_vec()).unwrap(); - let cache = InterpreterCache::new(); - let mut out = Vec::new(); - for page in pdf.pages().iter() { - let mut ctx = Context::new( - Affine::IDENTITY, - Rect::new(0.0, 0.0, 1.0, 1.0), - &cache, - pdf.xref(), - InterpreterSettings::default(), - ); - let mut glyphs = Glyphs(Vec::new()); - interpret_page(page, &mut ctx, &mut glyphs); - // PDF space runs up the page: the highest baseline first. - let mut lines: BTreeMap, Vec<(f64, String)>> = BTreeMap::new(); - for (y, x, t) in glyphs.0 { - lines.entry(std::cmp::Reverse(y)).or_default().push((x, t)); - } - for (_, mut line) in lines { - line.sort_by(|a, b| a.0.total_cmp(&b.0)); - out.push(line.into_iter().map(|(_, t)| t).collect()); - } - } - out + ledger_import::extract::pdf_lines(pdf).unwrap() } /// statement_pdf INVARIANT 4 (P3 review round 2): a company and accounts diff --git a/crates/ledger-mcp/handbook/evidence-and-receipts.md b/crates/ledger-mcp/handbook/evidence-and-receipts.md index c46176e..d19ebd2 100644 --- a/crates/ledger-mcp/handbook/evidence-and-receipts.md +++ b/crates/ledger-mcp/handbook/evidence-and-receipts.md @@ -5,3 +5,8 @@ - Not receipt-checked yet: books.create_invoice, books.create_bill, books.receive_payment, books.pay_bill, books.run_payroll and books.remit_payroll_taxes post at once, with no preview and no receipt check. Attach their evidence (the invoice, the bill, the payment confirmation, the payroll register) right after, linked to the entry they return. - Receipt corrections: receipts can change, but nothing is erased. To replace one, attach the new receipt, then books.unlink_receipt the old (with a reason); retire unwanted receipts or documents with books.withdraw_attachment (with a reason). The journal keeps every step. - File a receipt once and link it wherever it is evidence (the attachments section). +- **Reading a file.** Call books.get_receipt first: its default answer gives what the file is, the first 8 KiB of the text it was read as, and how many pages it has. Read the text (`view text` gives all of it). Open a page picture (`view page_image`, `page` 1 for the first) only when the text is empty or looks wrong on a page you must check, and never ask for `view bytes` into your context: a file goes to the human as a download, not through you (D-46). A picture-only receipt has no text by design (no OCR): read its pages. +- **When a file's text failed.** A PDF whose reading failed is listed in Needs attention; read its pages and, if you need the words, ask the human. Do not retype or re-save a file to get it past a check. +- **The same file twice.** Filing bytes the books already hold returns the existing attachment and links it to the new subject: one record, several subjects. Older books may hold two records of one file; `books.consolidate_duplicates` previews them (the oldest is kept, linked where the others were, and the rest withdrawn as duplicates, nothing deleted). It refuses whole rather than leave a posted entry the policy covers with no evidence. Annotations of a withdrawn duplicate are not moved: annotate the kept file. +- **Annotating a receipt.** Record what you read off it with books.annotate_attachment: the receipt number, payee, amount (a decimal string), date, instrument, category, source, who is to be reimbursed, and a note. It changes no amount and no file. Receipt numbers run from the highest one annotated: the answer's `next_receipt_number` is the one to give the next new receipt, so numbers are never reused. +- **Who paid, and who was paid back.** An expense a member paid personally is the company borrowing from them and spending it (the cash flow statement shows both). When a payment pays them back, record what it settles with books.set_entry_settles on the payment, naming the entries it pays back; Needs attention asks until you do, once per entry. Never guess which expense a payment covered (no oldest-first rule): ask the human. diff --git a/crates/ledger-mcp/handbook/format-upgrade.md b/crates/ledger-mcp/handbook/format-upgrade.md index e2de791..aff3758 100644 --- a/crates/ledger-mcp/handbook/format-upgrade.md +++ b/crates/ledger-mcp/handbook/format-upgrade.md @@ -7,3 +7,5 @@ - Before it upgrades, the server makes a backup of the books, their vault and keys included, and proves it whole; a backup that can't be made or proven upgrades nothing. Then the books move to the new format in one step, on the record. - After the upgrade an older version of wanderbooks can't open these books; the backup made first keeps them as they were. Going back is the owner's choice, never yours. - This upgrade is `documents_v2`: invoice numbers, terms and line accounts (S4), sent records (S5), payments that settle several documents with early-payment discounts and a processor's fee (S6), and write-offs (S7). Until it is given, those writes are refused and everything else works as before. + +- The journal features an upgrade can add: `members_cumulative`, `account_renamed`, `require_tax_category`, `tax_categories_gain_exempt`, `disposal_before_service`, `depreciation_catch_up`, `party_creditors`, `payment_instruments`, `vendor_1099_decisions`, `schedule_convention`, `month_reposting`, `asset_basis`, `projects`, `document_line_projects`, `document_standing`, `documents_v2`, and `attachment_text` (P6: a filed file's extracted text, what a human annotated of it, and the entries a payment settles). All are declared at once; a version that doesn't know one stops at the declaration instead of misreading the books. diff --git a/crates/ledger-mcp/src/admin.rs b/crates/ledger-mcp/src/admin.rs index d484278..0adb04b 100644 --- a/crates/ledger-mcp/src/admin.rs +++ b/crates/ledger-mcp/src/admin.rs @@ -165,6 +165,15 @@ pub enum Request { /// The owner passphrase, checked by the server (INVARIANT 7). passphrase: Passphrase, }, + /// Read one attachment's text now and journal it (plan P6 step 2): + /// the backfill, one attachment at a time, so a long books dir can be + /// resumed. + Extract { + /// The books. + id: String, + /// The attachment to read. + attachment_id: u64, + }, /// End a token. TokenRevoke { /// Its books. @@ -477,6 +486,18 @@ pub enum Response { }, /// A format upgrade made (or none needed). FormatUpgraded(FormatUpgraded), + /// An attachment's text read and journaled (plan P6 step 2). + Extracted { + /// The attachment. + attachment_id: u64, + /// "ok", "none" or "failed". + status: String, + /// How many pages it has. + pages: u32, + /// The text blob's SHA-256, when there is text. + #[serde(default, skip_serializing_if = "Option::is_none")] + text_sha256: Option, + }, /// Refused, with why (no path in the data root: INVARIANT 3). Refused(String), } @@ -530,6 +551,9 @@ pub fn call_within( Request::UpgradeFormat { .. } => { Some("`wanderbooks books list` says whether the books still wait for it") } + Request::Extract { .. } => Some( + "`wanderbooks books extract BOOK ATTACHMENT` reads it again; it journals once per extractor version", + ), }; let unknown = move |why: String| match look { None => CallError::Unknown(why), diff --git a/crates/ledger-mcp/src/local.rs b/crates/ledger-mcp/src/local.rs index fbcdc9f..a57cfb5 100644 --- a/crates/ledger-mcp/src/local.rs +++ b/crates/ledger-mcp/src/local.rs @@ -497,6 +497,34 @@ impl Books { expect_head, } => self.restore(id, source, replace, confirmed, expect_head), Request::UpgradeFormat { id, passphrase } => self.upgrade_format(id, &passphrase), + Request::Extract { id, attachment_id } => { + let id = match BookId::parse(&id) { + Ok(id) => id, + Err(e) => return refused(e), + }; + let store = match self.held().get(id.as_str()) { + Some(Book::Open { store, .. }) => store.clone(), + Some(Book::Failed(why)) => { + return Response::Refused(format!("those books didn't open: {why}")); + } + None => { + return Response::Refused(format!( + "there are no books called {:?}", + id.as_str() + )); + } + }; + match store.with_store_mut(|s| s.extract_attachment(attachment_id)) { + Ok(Ok(x)) => Response::Extracted { + attachment_id: x.attachment_id, + status: x.status.key().to_string(), + pages: x.pages, + text_sha256: x.text_sha256, + }, + Ok(Err(e)) => Response::Refused(self.hide(e.to_string())), + Err(why) => Response::Refused(self.hide(why)), + } + } Request::BackUpNow { id, new_disk } => { let id = match BookId::parse(&id) { Ok(id) => id, diff --git a/crates/ledger-mcp/src/scopes.rs b/crates/ledger-mcp/src/scopes.rs index 69b5a3e..f30f597 100644 --- a/crates/ledger-mcp/src/scopes.rs +++ b/crates/ledger-mcp/src/scopes.rs @@ -56,7 +56,7 @@ const C: Needs = PostWhenConfirmed; const K: Needs = PostForApprovalRules; /// Every tool and what it needs (INVARIANT 1). -pub const TABLE: [(&str, Needs); 125] = [ +pub const TABLE: [(&str, Needs); 129] = [ ("books.handbook", R), ("books.start", R), // Settings are committed at once: the settings commit of the plan. @@ -191,6 +191,11 @@ pub const TABLE: [(&str, Needs); 125] = [ ("books.back_up_now", A), // P4b S7: a bad-debt write-off (preview, then post). ("books.write_off_invoice", C), + // P6: evidence (plan/served.md P6). + ("books.find_attachment", R), + ("books.annotate_attachment", W), + ("books.set_entry_settles", W), + ("books.consolidate_duplicates", C), ]; /// What `tool` needs, given its arguments; None for a tool this server diff --git a/crates/ledger-mcp/src/server.rs b/crates/ledger-mcp/src/server.rs index 00f3ec7..d3e0155 100644 --- a/crates/ledger-mcp/src/server.rs +++ b/crates/ledger-mcp/src/server.rs @@ -576,6 +576,7 @@ impl BooksServer { + Self::reconciliation_router() + Self::parties_router() + Self::attachments_router() + + Self::evidence_router() + Self::settings_router() + Self::chart_router() + Self::payroll_router() @@ -1407,6 +1408,8 @@ mod chart; mod documents; #[path = "tools/entries.rs"] mod entries; +#[path = "tools/evidence.rs"] +mod evidence; #[path = "tools/helpers.rs"] mod helpers; #[path = "tools/parties.rs"] @@ -1442,44 +1445,58 @@ pub(crate) use helpers::*; // ServerHandler // --------------------------------------------------------------------------- +/// What a resource read gives: JSON/Markdown text, or a file's bytes (a +/// receipt's page picture, plan P6). +enum ResourceBody { + /// Text the agent reads. + Text(String), + /// Bytes, base64 in the MCP answer. + Blob { mime: String, bytes: Vec }, +} + impl BooksServer { - /// Render a resource URI to its JSON text. Used by `read_resource`. - async fn resource_text(&self, uri: &str) -> Result { + /// Render a resource URI to what it gives. Used by `read_resource`. + async fn resource_body(&self, uri: &str) -> Result { // Split off the query string for URIs like books://digest?since=... let (path, query) = match uri.split_once('?') { Some((p, q)) => (p, q), None => (uri, ""), }; + fn json(v: &T) -> Result { + serde_json::to_string_pretty(v) + .map(ResourceBody::Text) + .map_err(|e| e.to_string()) + } match path { "books://start" => { let Json(out) = self.start(Parameters(kept::StartParams::default())).await?; - serde_json::to_string_pretty(&out).map_err(|e| e.to_string()) + json(&out) } "books://settings" => { let Json(out) = self.settings_tool().await?; - serde_json::to_string_pretty(&out).map_err(|e| e.to_string()) + json(&out) } "books://conventions" => { let Json(out) = self .conventions(Parameters(kept::ConventionsParams::default())) .await?; - serde_json::to_string_pretty(&out).map_err(|e| e.to_string()) + json(&out) } "books://chart" => { let Json(out) = self.chart().await?; - serde_json::to_string_pretty(&out).map_err(|e| e.to_string()) + json(&out) } "books://reports/trial-balance" => { let Json(out) = self .trial_balance(Parameters(TrialBalanceParams::default())) .await?; - serde_json::to_string_pretty(&out).map_err(|e| e.to_string()) + json(&out) } "books://reports/balance-sheet" => { let Json(out) = self .balance_sheet(Parameters(BalanceSheetParams { as_of: None })) .await?; - serde_json::to_string_pretty(&out).map_err(|e| e.to_string()) + json(&out) } "books://reports/profit-and-loss" => { let Json(out) = self @@ -1489,16 +1506,16 @@ impl BooksServer { class: None, })) .await?; - serde_json::to_string_pretty(&out).map_err(|e| e.to_string()) + json(&out) } "books://digest" => { let since = digest_since(query, today()); let Json(out) = self.digest(Parameters(DigestParams { since })).await?; - serde_json::to_string_pretty(&out).map_err(|e| e.to_string()) + json(&out) } handbook::URI => { let Json(out) = self.handbook(Parameters(HandbookParams::default())).await?; - serde_json::to_string_pretty(&out).map_err(|e| e.to_string()) + json(&out) } _ if path.starts_with(&format!("{}/", handbook::URI)) => { let id = &path[handbook::URI.len() + 1..]; @@ -1508,21 +1525,59 @@ impl BooksServer { self.currency(), self.document_line_projects(), ) + .map(ResourceBody::Text) .ok_or_else(|| format!("unknown handbook section: {id}")) } - _ if path.starts_with("books://receipts/") => { - let id: u64 = path - .strip_prefix("books://receipts/") - .unwrap_or("") + _ if (path.starts_with("books://receipts/") + || path.starts_with("books://attachments/")) => + { + // `books://attachments/{id}/page/{n}`: one page as a PNG + // (plan P6; D-46: a file is handed over, but a resource + // read is the agent's own fetch of a picture, never a + // path). + let prefix = if path.starts_with("books://receipts/") { + "books://receipts/" + } else { + "books://attachments/" + }; + let rest = &path[prefix.len()..]; + if let Some((id, page)) = rest.split_once("/page/") { + let id: u64 = id + .parse() + .map_err(|_| format!("bad attachment resource URI: {uri}"))?; + let page: u32 = page + .parse() + .map_err(|_| format!("bad page in {uri}: give a page number from 1"))?; + if page == 0 { + return Err("page numbers start at 1".into()); + } + let store = self.lock()?; + let meta = find_attachment(&store, id)?; + let bytes = store.attachment_bytes(id).map_err(|e| e.to_string())?; + let png = ledger_import::extract::page_png( + &bytes, + usize::try_from(page - 1).unwrap_or(usize::MAX), + 1600, + ) + .map_err(|e| e.to_string())?; + let _ = meta; + return Ok(ResourceBody::Blob { + mime: "image/png".into(), + bytes: png, + }); + } + let id: u64 = rest .parse() - .map_err(|_| format!("bad receipt resource URI: {uri}"))?; + .map_err(|_| format!("bad attachment resource URI: {uri}"))?; let Json(out) = self .get_receipt(Parameters(GetReceiptParams { attachment_id: id, view: None, + page: None, + max_px: None, })) .await?; - serde_json::to_string_pretty(&out).map_err(|e| e.to_string()) + json(&out) } _ => Err(format!("unknown resource: {uri}")), } @@ -1842,29 +1897,43 @@ impl BooksServer { let uri = request.uri.clone(); let server = self.clone(); let at = uri.clone(); - let text = run_blocking(&self.store, move || { - Box::pin(async move { in_timezone_async(timezone, server.resource_text(&at)).await }) + let body = run_blocking(&self.store, move || { + Box::pin(async move { in_timezone_async(timezone, server.resource_body(&at)).await }) }) .await?; // A refusal quotes the URI it was given: masked like a tool's. - let text = text.map_err(|e| { + let body = body.map_err(|e| { redact_error(rmcp::ErrorData::resource_not_found( uri.clone(), Some(serde_json::json!({"reason": e})), )) })?; - let mime_type = if uri.starts_with(&format!("{}/", handbook::URI)) { - "text/markdown" - } else { - "application/json" + let contents = match body { + ResourceBody::Text(text) => { + let mime_type = if uri.starts_with(&format!("{}/", handbook::URI)) { + "text/markdown" + } else { + "application/json" + }; + ResourceContents::TextResourceContents { + uri, + mime_type: Some(mime_type.to_string()), + text, + meta: None, + } + } + ResourceBody::Blob { mime, bytes } => { + use base64::Engine; + ResourceContents::BlobResourceContents { + uri, + mime_type: Some(mime), + blob: base64::engine::general_purpose::STANDARD.encode(bytes), + meta: None, + } + } }; Ok(ReadResourceResponse::Complete(ReadResourceResult::new( - vec![ResourceContents::TextResourceContents { - uri, - mime_type: Some(mime_type.to_string()), - text, - meta: None, - }], + vec![contents], ))) } } diff --git a/crates/ledger-mcp/src/server_tests.rs b/crates/ledger-mcp/src/server_tests.rs index ada157d..7e918cc 100644 --- a/crates/ledger-mcp/src/server_tests.rs +++ b/crates/ledger-mcp/src/server_tests.rs @@ -42,7 +42,7 @@ fn annotations_complete() { const D: Hints = (Some(false), Some(true), None); const WI: Hints = (Some(false), Some(false), Some(true)); const DI: Hints = (Some(false), Some(true), Some(true)); - let table: [(&str, Hints); 125] = [ + let table: [(&str, Hints); 129] = [ ("books.handbook", R), ("books.start", R), ("books.set_setting", DI), @@ -164,6 +164,11 @@ fn annotations_complete() { ("books.project", R), ("books.profit_and_loss_by_class", R), ("books.export_cpa_package", W), + // P6: evidence (plan/served.md P6). + ("books.find_attachment", R), + ("books.annotate_attachment", W), + ("books.set_entry_settles", W), + ("books.consolidate_duplicates", D), ("books.access_log", R), ("books.backup", R), ("books.back_up_now", WI), @@ -208,8 +213,8 @@ fn annotations_complete() { "{name}: openWorldHint" ); } - // docs/mcp-interface.md names the 30 destructive tools. - assert_eq!(table.iter().filter(|(_, h)| h.1 == Some(true)).count(), 30); + // docs/mcp-interface.md names the 31 destructive tools. + assert_eq!(table.iter().filter(|(_, h)| h.1 == Some(true)).count(), 31); } /// Every refusal an agent can be given whose text is fixed here, or @@ -472,6 +477,12 @@ fn no_tool_takes_a_path() { &mut props, ); for p in props { + // An annotation's `source` (plan P6 step 5) is what a human + // says a receipt came from ("the owner's email"), kept as text + // and never read as a place on this computer. + if t.name == "books.annotate_attachment" && p == "source" { + continue; + } let lower = p.to_ascii_lowercase(); // Names that hold a place to read or write: whole words, // so `link_to`, `*_file_name`, `tin_on_file` and `date_to` diff --git a/crates/ledger-mcp/src/tin_files.rs b/crates/ledger-mcp/src/tin_files.rs index 96e941b..00cc29b 100644 --- a/crates/ledger-mcp/src/tin_files.rs +++ b/crates/ledger-mcp/src/tin_files.rs @@ -157,7 +157,7 @@ pub fn refuse( if holds_sha(vault_dir, &ledger_store::journal::sha256_hex(bytes))? { return Err(TIN_FILE.to_string()); } - let Some(text) = text_of(bytes, mime_type, file_name) else { + let Some(text) = ledger_import::extract::text_of(bytes, mime_type, file_name) else { return Ok(()); }; let text: Vec = text.chars().map(fold).collect(); @@ -278,97 +278,10 @@ fn holds_printed_tin(text: &[char], own_ein: Option) -> bool { }) } -/// Whether a file of this type or name is text: csv, tsv, txt, json, iif, -/// xml, html, md and their kin. -fn text_like(mime_type: &str, file_name: &str) -> bool { - let mime = mime_type.trim().to_ascii_lowercase(); - let ext = Path::new(file_name) - .extension() - .and_then(|e| e.to_str()) - .map(str::to_ascii_lowercase) - .unwrap_or_default(); - mime.starts_with("text/") - || [ - "json", - "xml", - "csv", - "x-iif", - "javascript", - "x-yaml", - "yaml", - ] - .iter() - .any(|t| mime.ends_with(&format!("/{t}")) || mime.ends_with(&format!("+{t}"))) - || [ - "csv", "tsv", "txt", "text", "json", "iif", "xml", "html", "htm", "md", "markdown", - "yaml", "yml", "tab", "prn", "log", "ofx", "qfx", "qif", - ] - .contains(&ext.as_str()) -} - -/// The bytes as text, if they are text (INVARIANT 3): UTF-16 with its -/// byte-order mark, UTF-8, or, for a text type or name, whatever they -/// decode to. Text with NUL bytes and no byte-order mark (UTF-16 written -/// without one is valid UTF-8, its digits split by NULs) is read every -/// way it can be (as it is, as UTF-16 both ways round, and with its NULs -/// taken out), each reading on a line of its own: so is bytes that aren't -/// UTF-8 but are NUL-interleaved as UTF-16 is. -fn text_of(bytes: &[u8], mime_type: &str, file_name: &str) -> Option { - let utf16 = |body: &[u8], le: bool| { - let units: Vec = body - .as_chunks::<2>() - .0 - .iter() - .map(|&c| { - if le { - u16::from_le_bytes(c) - } else { - u16::from_be_bytes(c) - } - }) - .collect(); - String::from_utf16_lossy(&units) - }; - match bytes { - [0xFF, 0xFE, body @ ..] => Some(utf16(body, true)), - [0xFE, 0xFF, body @ ..] => Some(utf16(body, false)), - _ => { - let plain = match std::str::from_utf8(bytes) { - Ok(t) => Some(t.to_string()), - Err(_) if text_like(mime_type, file_name) => { - Some(String::from_utf8_lossy(bytes).into_owned()) - } - Err(_) => None, - }; - if !bytes.contains(&0) || plain.is_none() && !nul_interleaved(bytes) { - return plain; - } - let without_nuls: Vec = bytes.iter().copied().filter(|&b| b != 0).collect(); - let readings = [ - plain.unwrap_or_default(), - utf16(bytes, true), - utf16(bytes, false), - String::from_utf8_lossy(&without_nuls).into_owned(), - ]; - Some(readings.join("\n")) - } - } -} - -/// Whether `bytes` look like UTF-16 without a byte-order mark: of their -/// byte pairs, at least half have a NUL on one side (the high byte of a -/// Latin character) and at most an eighth on the other. -fn nul_interleaved(bytes: &[u8]) -> bool { - let pairs = bytes.as_chunks::<2>().0; - let nuls = |side: usize| pairs.iter().filter(|p| p[side] == 0).count(); - let (first, second) = (nuls(0), nuls(1)); - let n = pairs.len(); - n > 0 && (2 * first >= n && 8 * second <= n || 2 * second >= n && 8 * first <= n) -} - #[cfg(test)] mod tests { use super::*; + use ledger_import::extract::nul_interleaved; /// A test's dir, removed when the test ends, pass or panic. struct TestDir(std::path::PathBuf); diff --git a/crates/ledger-mcp/src/tools/attachments.rs b/crates/ledger-mcp/src/tools/attachments.rs index a9d0121..9b6de1c 100644 --- a/crates/ledger-mcp/src/tools/attachments.rs +++ b/crates/ledger-mcp/src/tools/attachments.rs @@ -138,7 +138,7 @@ impl BooksServer { #[tool( name = "books.file_document", title = "File a document", - description = "File a document: category formation, legal, tax, accounting, statement (a bank or card statement: then books.record_statement), contract, photo, warranty, import_source (a bank export), or other. link_to links it to subjects in the same call (an account, an asset, a party, a project, the company: see books.link_attachment), all checked first. The bytes: upload_id, from books.begin_upload and `wanderbooks upload` (a file never passes through you). Never a W-9 (books.set_w9 w9_upload_id puts it in the vault). A file holding a full TIN is refused: one the server read or wrote with TINs, or text with a TIN the vault keeps or an SSN or EIN printed 3-2-4 or 2-7 with hyphens.", + description = "File a document: category formation, legal, tax, accounting, statement (then books.record_statement), contract, photo, warranty, import_source (a bank export), notices, correspondence, plans, history, or other. link_to links it to subjects in the same call (see books.link_attachment), all checked first. The bytes: upload_id, from books.begin_upload and `wanderbooks upload` (a file never passes through you). Never a W-9 (books.set_w9). A file holding a full TIN is refused, as is text with one the vault keeps or an SSN or EIN printed 3-2-4 or 2-7.", annotations( read_only_hint = false, destructive_hint = false, @@ -152,10 +152,11 @@ impl BooksServer { let p = params.0; match p.category.as_str() { "formation" | "legal" | "tax" | "accounting" | "statement" | "contract" | "photo" - | "warranty" | "import_source" | "other" => {} + | "warranty" | "import_source" | "notices" | "correspondence" | "plans" | "history" + | "other" => {} _ => { return Err(format!( - "bad category {:?}: expected formation, legal, tax, accounting, statement, contract, photo, warranty, import_source, or other", + "bad category {:?}: expected formation, legal, tax, accounting, statement, contract, photo, warranty, import_source, notices, correspondence, plans, history, or other", p.category )); } @@ -454,7 +455,7 @@ impl BooksServer { #[tool( name = "books.get_receipt", title = "Get a receipt", - description = "A receipt or document: what it is and everything it is linked to. view bytes adds the file, handed over: run its download (wanderbooks download) to save a copy in a folder you name; its bytes never come back here.", + description = "A receipt or document: what it is, what it is linked to, its first 8 KiB of text and its pages. view text gives the whole extracted text (past 64 KiB as a handed-over text file); view page_image with page (1 is the first) gives one page as a PNG, handed over (D-46, D-15: no OCR). view bytes hands over the file itself. Text first; a page picture only when the text is empty or wrong; never read a file's bytes into your context.", annotations(read_only_hint = true, open_world_hint = false) )] pub(super) async fn get_receipt( @@ -462,29 +463,90 @@ impl BooksServer { params: Parameters, ) -> Result, String> { let p = params.0; - let bytes_wanted = match p.view.as_deref().unwrap_or("meta") { - "meta" => false, - "bytes" => true, - other => { - return Err(format!( - "view {:?}: meta (the default) or bytes", - other.chars().take(20).collect::() - )); - } - }; + let view = p.view.as_deref().unwrap_or("meta"); + if !matches!(view, "meta" | "text" | "page_image" | "bytes") { + return Err(format!( + "view {:?}: meta (the default), text, page_image or bytes", + view.chars().take(20).collect::() + )); + } let store = self.lock()?; let meta = find_attachment(&store, p.attachment_id)?; + let extraction = store.extraction(meta.id).map_err(|e| e.to_string())?; + let pages = extraction.as_ref().map_or(0, |x| x.pages); + let text_status = extraction + .as_ref() + .map_or("not_extracted", |x| x.status.key()) + .to_string(); + let annotation = store.ledger().annotation(meta.id).cloned(); + let full_text = store.extraction_text(meta.id).map_err(|e| e.to_string())?; + let book = book_name(&store); // Handed over, read hash-verified (D-46): never its bytes here. - let file = if bytes_wanted { - let bytes = store - .attachment_bytes(p.attachment_id) + let mut file = None; + // Every view carries the meta answer's first 8 KiB of text (view + // `text` replaces it with the whole text, or hands a file over). + let mut text = None; + let mut text_truncated = false; + if let Some(t) = &full_text { + let cut = floor_char_boundary(t, 8 * 1024); + text_truncated = cut < t.len(); + text = Some(t[..cut].to_string()); + } + match view { + "meta" => {} + "text" => match full_text { + None => { + return Err(format!( + "attachment {} has no extracted text (text_status {text_status}); view page_image to read its pages", + meta.id + )); + } + Some(t) if t.len() <= 64 * 1024 => text = Some(t), + Some(t) => { + // Past 64 KiB the whole text is the handed file, not + // the preview. + text = None; + text_truncated = false; + let name = format!("{}-text.txt", stem(&meta.file_name)); + file = Some(self.hand_over(&book, &name, "text/plain", t.into_bytes())?); + } + }, + "page_image" => { + let page = p.page.ok_or("view page_image needs page: 1 is the first")?; + let max_px = p.max_px.unwrap_or(1600); + if !(1..=8192).contains(&max_px) { + return Err("max_px must be 1 to 8192".into()); + } + let bytes = store.attachment_bytes(meta.id).map_err(|e| e.to_string())?; + if meta.mime_type != "application/pdf" && !bytes.starts_with(b"%PDF") { + return Err(format!( + "attachment {} isn't a PDF: view bytes for the file itself", + meta.id + )); + } + let png = ledger_import::extract::page_png( + &bytes, + usize::try_from(page - 1).unwrap_or(usize::MAX), + max_px, + ) .map_err(|e| e.to_string())?; - Some(self.hand_over(&book_name(&store), &meta.file_name, &meta.mime_type, bytes)?) - } else { - None - }; + let name = format!("{}-p{page}.png", stem(&meta.file_name)); + file = Some(self.hand_over(&book, &name, "image/png", png)?); + } + "bytes" => { + let bytes = store.attachment_bytes(meta.id).map_err(|e| e.to_string())?; + file = Some(self.hand_over(&book, &meta.file_name, &meta.mime_type, bytes)?); + } + _ => unreachable!("view checked"), + } Ok(Json(GetReceiptOut { file, + text, + text_truncated, + text_status, + pages, + receipt_number: annotation.as_ref().and_then(|a| a.fields.receipt_number), + annotation: annotation.as_ref().map(annotation_out), links: links_out(&store, meta.id)?, description: meta.description.clone(), attachment_id: meta.id, diff --git a/crates/ledger-mcp/src/tools/evidence.rs b/crates/ledger-mcp/src/tools/evidence.rs new file mode 100644 index 0000000..8f717d3 --- /dev/null +++ b/crates/ledger-mcp/src/tools/evidence.rs @@ -0,0 +1,412 @@ +//! Evidence beyond a file's bytes: finding it, annotating it, and +//! consolidating duplicates (plan/served.md P6 steps 4 and 5). +//! +//! ```text +//! GOAL: a receipt found by what it is or says, the facts a human read off +//! it kept on the record, and identical files held once, linked +//! wherever they are evidence. +//! SOURCE: plan/served.md P6 step 4 (defect 4) and step 5 (defect 10); +//! IZ-18 (no FIFO guess: `entry.settles` only); Rev. Proc. 98-25 +//! (an audit trail); rules.md B-44. +//! ``` + +use super::*; + +/// A date as the tool takes it. +fn date_of(text: &str) -> Result { + NaiveDate::parse_from_str(text.trim(), "%Y-%m-%d") + .map_err(|_| format!("{text:?} isn't a date: give YYYY-MM-DD")) +} + +#[tool_router(router = evidence_router, vis = "pub(super)")] +impl BooksServer { + /// ```text + /// GOAL: find a filed file by its exact bytes, its receipt number, or + /// what its text says. + /// SOURCE: plan/served.md P6 step 4; B-44. + /// INVARIANTS: + /// 1. EXACTLY one of sha256, receipt_number and text is given. + /// 2. Withdrawn files are listed too, marked (a duplicate is found + /// before it is consolidated). + /// 3. A text search is case-insensitive over the extracted text. + /// SCENARIOS: + /// 1. IF none or more than one is given: SHOULD refuse. + /// 2. IF sha256 is not 64 hex digits: SHOULD refuse. + /// 3. IF nothing matches: SHOULD answer with an empty list. + /// ``` + #[tool( + name = "books.find_attachment", + title = "Find a filed file", + description = "Find receipts and documents by exactly one of: sha256 (the file's exact hash), receipt_number (what books.annotate_attachment recorded), or text (words its extracted text says, case-insensitively). Withdrawn files are listed too, marked, so a duplicate is found before books.consolidate_duplicates. Each comes with all its links.", + annotations(read_only_hint = true, open_world_hint = false) + )] + async fn find_attachment( + &self, + params: Parameters, + ) -> Result, String> { + let p = params.0; + let given = [ + p.sha256.is_some(), + p.receipt_number.is_some(), + p.text.is_some(), + ] + .into_iter() + .filter(|g| *g) + .count(); + if given != 1 { + return Err( + "give exactly one of sha256, receipt_number or text: they find a file three ways" + .into(), + ); + } + let store = self.lock()?; + let mut found = if let Some(sha) = p.sha256 { + let sha = sha.trim().to_lowercase(); + if sha.len() != 64 || !sha.bytes().all(|b| b.is_ascii_hexdigit()) { + return Err("sha256 is 64 hexadecimal digits".into()); + } + store + .list_attachments(None) + .map_err(|e| e.to_string())? + .into_iter() + .filter(|a| a.blob_sha.eq_ignore_ascii_case(&sha)) + .collect() + } else if let Some(n) = p.receipt_number { + store + .attachments_with_receipt_number(n) + .map_err(|e| e.to_string())? + } else { + let text = p.text.unwrap_or_default(); + let text = text.trim(); + if text.chars().count() < 2 { + return Err("text is at least two characters".into()); + } + store + .attachments_holding_text(text) + .map_err(|e| e.to_string())? + }; + found.sort_by_key(|a| a.id); + let attachments = found + .into_iter() + .map(|a| receipt_out(&store, a)) + .collect::>()?; + Ok(Json(FindAttachmentOut { attachments })) + } + + /// ```text + /// GOAL: record what a human read off a file, without touching the file + /// or the ledger's amounts. + /// SOURCE: plan/served.md P6 step 5 (defect 10); B-44. + /// INVARIANTS: + /// 1. Journaled as `attachment.annotated`; a refusal journals nothing. + /// 2. The fields are the closed list of `AnnotationFieldsParam`; at + /// least one field or the note is given. + /// 3. A new annotation replaces the file's earlier one; the file + /// stays filed and its bytes unchanged. + /// 4. The answer names the next receipt number (one past the highest + /// annotated). + /// SCENARIOS: + /// 1. IF the attachment is unknown or withdrawn: SHOULD refuse. + /// 2. IF nothing is given, an amount is negative or malformed, or a + /// date is outside the calendar: SHOULD refuse. + /// ``` + #[tool( + name = "books.annotate_attachment", + title = "Annotate a filed file", + description = "Record what a human read off a receipt or document: receipt_number, payee, amount (decimal string), date (YYYY-MM-DD), instrument, category, source, reimbursable_to, and a note. It changes no amount and no file; a later annotation replaces this one. The answer gives next_receipt_number (one past the highest annotated), so receipt numbers run without reuse. Amounts are exact decimal strings.", + annotations( + read_only_hint = false, + destructive_hint = false, + open_world_hint = false + ) + )] + async fn annotate_attachment( + &self, + params: Parameters, + ) -> Result, String> { + let p = params.0; + let currency = self.currency(); + let amount = match p.fields.amount.as_deref() { + None => None, + Some(a) => Some(a.trim().parse::().map_err(|_| { + format!("{a:?} isn't an amount: give a decimal string like \"12.34\"") + })?), + }; + let date = match p.fields.date.as_deref() { + None => None, + Some(d) => Some(date_of(d)?), + }; + let annotation = ledger_core::Annotation { + attachment_id: p.attachment_id, + fields: ledger_core::AnnotationFields { + receipt_number: p.fields.receipt_number, + payee: p.fields.payee, + amount, + date, + instrument: p.fields.instrument, + category: p.fields.category, + source: p.fields.source, + reimbursable_to: p.fields.reimbursable_to, + }, + note: p.note, + }; + let _ = currency; + let mut store = self.lock()?; + let stored = store + .annotate_attachment(annotation) + .map_err(|e| e.to_string())?; + let next = store.ledger().next_receipt_number(); + Ok(Json(AnnotateAttachmentOut { + attachment_id: stored.attachment_id, + fields: annotation_out(&stored), + next_receipt_number: next, + })) + } + + /// ```text + /// GOAL: record which entries a payment settles, so "the member was + /// paid back" is read from the books and never guessed. + /// SOURCE: plan/served.md P6 step 5 (IZ-18: no FIFO guess). + /// INVARIANTS: + /// 1. Journaled as `entry.settles`; a refusal journals nothing. + /// 2. Every named entry exists, none is the paying entry itself, none + /// is named twice. + /// 3. An empty list clears the record. + /// SCENARIOS: + /// 1. IF the paying entry or a settled entry is unknown, or the list + /// names the paying entry or one twice: SHOULD refuse. + /// ``` + #[tool( + name = "books.set_entry_settles", + title = "Record what a payment settles", + description = "Record which entries an entry pays back, such as a reimbursement to a member settling the expenses they paid personally. Needs attention stops asking about a member-paid entry once a payment names it here. Empty clears the record. It changes no amount: it names what the payment settles.", + annotations( + read_only_hint = false, + destructive_hint = false, + open_world_hint = false + ) + )] + async fn set_entry_settles( + &self, + params: Parameters, + ) -> Result, String> { + let p = params.0; + let mut store = self.lock()?; + store + .set_entry_settles(p.entry_id, &p.settles) + .map_err(|e| e.to_string())?; + Ok(Json(SetEntrySettlesOut { + ok: true, + entry_id: p.entry_id, + settles: p.settles, + })) + } + + /// ```text + /// GOAL: hold identical files once, linked wherever they were evidence, + /// without ever leaving an entry the receipt policy covers with + /// none. + /// SOURCE: plan/served.md P6 step 4 (defect 4); rules.md B-44. + /// INVARIANTS: + /// 1. Records sharing one blob SHA are one set; the oldest is kept + /// and linked to every subject the others had. + /// 2. Every other record is unlinked, then withdrawn as a duplicate + /// (`duplicate_of`), journaled; nothing is deleted. + /// 3. Every touched entry's evidence is re-checked before the first + /// journal append: an entry that would be left with none and + /// needs one refuses the whole call, journaling nothing. + /// 4. Without `confirmed`, a preview that changes nothing. + /// SCENARIOS: + /// 1. IF the kept record is withdrawn and cannot stand in for a + /// duplicate that is a posted entry's only evidence: SHOULD + /// refuse (receipt_required). + /// 2. IF no two records share bytes: SHOULD answer with no groups. + /// ``` + #[tool( + name = "books.consolidate_duplicates", + title = "Consolidate duplicate files", + description = "Find receipts and documents filed more than once with the same bytes. For each set it keeps the oldest, links it to every subject the others had, unlinks the others, and withdraws them as duplicates of it (nothing is deleted). Refused whole when it would leave a posted entry the receipt policy covers with no evidence. Without confirmed, a preview that changes nothing.", + annotations( + read_only_hint = false, + destructive_hint = true, + open_world_hint = false + ) + )] + async fn consolidate_duplicates( + &self, + params: Parameters, + ) -> Result, String> { + use std::collections::{BTreeMap, BTreeSet}; + let confirmed = params.0.confirmed.unwrap_or(false); + let mut store = self.lock()?; + let all = store.list_attachments(None).map_err(|e| e.to_string())?; + let mut by_sha: BTreeMap> = BTreeMap::new(); + for a in all { + by_sha.entry(a.blob_sha.clone()).or_default().push(a); + } + // One plan per set of two or more records. + struct Group { + sha: String, + keep: ledger_store::Attachment, + dups: Vec, + link: BTreeSet, + unlink: Vec<(u64, ledger_store::Subject)>, + } + let mut groups: Vec = Vec::new(); + for (sha, mut records) in by_sha { + if records.len() < 2 { + continue; + } + records.sort_by_key(|a| a.id); + let keep = records.remove(0); + let mut link: BTreeSet = BTreeSet::new(); + let mut unlink = Vec::new(); + let mut live_dups = Vec::new(); + for d in records { + if d.withdrawn_at.is_some() { + continue; + } + for sub in subjects_of(&store, &d)? { + if !covered(&store, &keep, sub)? { + link.insert(sub); + } + if !matches!(sub, ledger_store::Subject::Entry(e) if Some(e) == d.entry_id) { + unlink.push((d.id, sub)); + } + } + live_dups.push(d); + } + if live_dups.is_empty() { + continue; + } + groups.push(Group { + sha, + keep, + dups: live_dups, + link, + unlink, + }); + } + // Consolidating writes `duplicate_of`: the books must read the + // feature before anything at all is journaled (plan P6, R-A). + if !groups.is_empty() { + store + .ensure_feature(ledger_store::format::ATTACHMENT_TEXT) + .map_err(|e| e.to_string())?; + } + // The evidence gate, before anything is journaled: an entry that + // would be left with none, and needs one, refuses the whole call. + for g in &groups { + let keep_stands = g.keep.withdrawn_at.is_none(); + for sub in &g.link { + if let ledger_store::Subject::Entry(e) = sub { + let mut left = store + .list_attachments(Some(*e)) + .map_err(|e| e.to_string())? + .into_iter() + .filter(|a| a.withdrawn_at.is_none()) + .filter(|a| !g.dups.iter().any(|d| d.id == a.id)) + .count(); + if keep_stands && !covered(&store, &g.keep, *sub)? { + left += 1; + } + if left == 0 { + self.ensure_receipt_not_required(&store, *e) + .map_err(|why| { + format!( + "not consolidated: entry {e}'s only evidence is a duplicate, and the kept file can't stand in for it ({why})" + ) + })?; + } + } + } + } + let out_groups: Vec = groups + .iter() + .map(|g| DuplicateGroupOut { + sha256: g.sha.clone(), + keep: g.keep.id, + withdraw: g.dups.iter().map(|d| d.id).collect(), + links: g.link.iter().map(|s| subject_out(&store, *s)).collect(), + }) + .collect(); + let withdrawn = out_groups.iter().map(|g| g.withdraw.len()).sum(); + if !confirmed { + return Ok(Json(ConsolidateDuplicatesOut { + status: "preview".into(), + groups: out_groups, + withdrawn, + })); + } + for g in &groups { + for sub in &g.link { + store + .link_attachment(g.keep.id, *sub) + .map_err(|e| e.to_string())?; + } + let reason = format!("duplicate of #{}", g.keep.id); + for (id, sub) in &g.unlink { + // A link a previous step took off (it was on the kept file + // already) isn't there to take off again. + if store + .attachment_links(*id) + .map_err(|e| e.to_string())? + .contains(sub) + { + store + .unlink_attachment(*id, *sub, &reason) + .map_err(|e| e.to_string())?; + } + } + for d in &g.dups { + if d.entry_id.is_some() { + store + .unlink_receipt(d.id, &reason) + .map_err(|e| e.to_string())?; + } + store + .withdraw_attachment_duplicate(d.id, &reason, g.keep.id) + .map_err(|e| e.to_string())?; + } + } + Ok(Json(ConsolidateDuplicatesOut { + status: "consolidated".into(), + groups: out_groups, + withdrawn, + })) + } +} + +/// Every subject a filing is evidence of: the entry it is filed on, and +/// every explicit link. +fn subjects_of( + store: &Store, + a: &ledger_store::Attachment, +) -> Result, String> { + let mut out: Vec = a + .entry_id + .map(ledger_store::Subject::Entry) + .into_iter() + .collect(); + out.extend(store.attachment_links(a.id).map_err(|e| e.to_string())?); + out.sort(); + out.dedup(); + Ok(out) +} + +/// Whether `a` is already evidence of `sub`. +fn covered( + store: &Store, + a: &ledger_store::Attachment, + sub: ledger_store::Subject, +) -> Result { + if let ledger_store::Subject::Entry(e) = sub + && a.entry_id == Some(e) + { + return Ok(true); + } + Ok(store + .attachment_links(a.id) + .map_err(|e| e.to_string())? + .contains(&sub)) +} diff --git a/crates/ledger-mcp/src/tools/files.rs b/crates/ledger-mcp/src/tools/files.rs index 37a7ee5..0a6f5bf 100644 --- a/crates/ledger-mcp/src/tools/files.rs +++ b/crates/ledger-mcp/src/tools/files.rs @@ -48,13 +48,14 @@ pub(crate) const INLINE_ARGS: [&str; 5] = [ /// Why `args` can't be taken by `tool` (INVARIANT 5): an argument that /// carries a file's bytes or text. books.begin_upload's own sha256 is the -/// file's hash, not its bytes. +/// file's hash, not its bytes, and books.find_attachment's is what it +/// searches by (plan P6 step 4): neither file passes through the call. pub(crate) fn inline_refusal( tool: &str, args: Option<&serde_json::Map>, ) -> Option { let given = args?.keys().find(|k| INLINE_ARGS.contains(&k.as_str()))?; - if tool == "books.begin_upload" && given == "sha256" { + if matches!(tool, "books.begin_upload" | "books.find_attachment") && given == "sha256" { return None; } Some(format!( @@ -368,6 +369,11 @@ mod tests { inline_refusal("books.begin_upload", Some(&args("sha256"))), None ); + // A search by a file's hash names no bytes (plan P6). + assert_eq!( + inline_refusal("books.find_attachment", Some(&args("sha256"))), + None + ); assert!(inline_refusal("books.begin_upload", Some(&args("content_base64"))).is_some()); assert_eq!(inline_refusal("books.attach_receipt", None), None); } diff --git a/crates/ledger-mcp/src/tools/helpers.rs b/crates/ledger-mcp/src/tools/helpers.rs index a583bf7..4be05b6 100644 --- a/crates/ledger-mcp/src/tools/helpers.rs +++ b/crates/ledger-mcp/src/tools/helpers.rs @@ -726,3 +726,41 @@ pub(crate) fn sent_out(s: &ledger_core::arap::Sent) -> SentOut { pub(crate) fn last_sent(sends: &[ledger_core::arap::Sent]) -> Option { sends.iter().map(|s| s.on).max().map(|d| d.to_string()) } + +/// An annotation as a tool answers with it (plan P6 step 5). +pub(crate) fn annotation_out(a: &ledger_core::Annotation) -> AnnotationOut { + AnnotationOut { + receipt_number: a.fields.receipt_number, + payee: a.fields.payee.clone(), + amount: a.fields.amount.map(money), + date: a.fields.date.map(|d| d.to_string()), + instrument: a.fields.instrument.clone(), + category: a.fields.category.clone(), + source: a.fields.source.clone(), + reimbursable_to: a.fields.reimbursable_to.clone(), + note: a.note.clone(), + } +} + +/// A file name's stem, for a page picture's or a text file's name: what is +/// before the last dot, or "attachment" when there is nothing. +pub(crate) fn stem(file_name: &str) -> String { + let stem = file_name + .rsplit_once('.') + .map_or(file_name, |(stem, _)| stem) + .trim(); + if stem.is_empty() { + "attachment".to_string() + } else { + stem.chars().take(80).collect() + } +} + +/// The largest character boundary at or below `at` in `text`. +pub(crate) fn floor_char_boundary(text: &str, at: usize) -> usize { + let mut cut = at.min(text.len()); + while !text.is_char_boundary(cut) { + cut -= 1; + } + cut +} diff --git a/crates/ledger-mcp/src/types.rs b/crates/ledger-mcp/src/types.rs index 343edd3..6a17706 100644 --- a/crates/ledger-mcp/src/types.rs +++ b/crates/ledger-mcp/src/types.rs @@ -1849,11 +1849,21 @@ pub struct ListReceiptsOut { pub struct GetReceiptParams { /// Attachment id. pub attachment_id: u64, - /// What to return: "meta" (default: what it is and what it's linked - /// to), or "bytes" (also the file, handed over: the command that - /// saves a copy, never its bytes; D-46). + /// What to return: "meta" (default: what it is, what it's linked to, + /// the first 8 KiB of its text and its page count), "text" (the whole + /// extracted text, or a handed-over text file past 64 KiB), + /// "page_image" (one page as a PNG, handed over: give `page`), or + /// "bytes" (the file itself, handed over: the command that saves a + /// copy, never its bytes; D-46). #[serde(default)] pub view: Option, + /// For view "page_image": the page, 1 for the first. + #[serde(default)] + pub page: Option, + /// For view "page_image": the picture's longest side in pixels + /// (default 1600, at most 8192). + #[serde(default)] + pub max_px: Option, } /// Result of `books.get_receipt`: what it is and what it's linked to, and @@ -1878,10 +1888,31 @@ pub struct GetReceiptOut { pub description: Option, /// Everything it is linked to (the entry it is filed on first). pub links: Vec, - /// With view `bytes`: the file, handed over: run its `download` to - /// save a copy in a folder you name. + /// With view `bytes` or `page_image`: the file (or the page picture), + /// handed over: run its `download` to save a copy in a folder you + /// name. #[serde(skip_serializing_if = "Option::is_none")] pub file: Option, + /// The first 8 KiB of the extracted text (view `meta`), or the whole + /// text (view `text`, when it fits in 64 KiB). + #[serde(skip_serializing_if = "Option::is_none")] + pub text: Option, + /// True when `text` is only the first 8 KiB of a longer extraction. + #[serde(default)] + pub text_truncated: bool, + /// What reading the file found: "ok", "none", "failed", or + /// "not_extracted" (books this version didn't read yet: + /// `wanderbooks books extract`). + pub text_status: String, + /// How many pages the file has (0 when it has none). + #[serde(default)] + pub pages: u32, + /// The receipt number annotated on it, if any. + #[serde(skip_serializing_if = "Option::is_none")] + pub receipt_number: Option, + /// Everything annotated about it, if anything. + #[serde(skip_serializing_if = "Option::is_none")] + pub annotation: Option, } // --------------------------------------------------------------------------- @@ -5353,3 +5384,160 @@ pub struct WriteOffInvoiceOut { /// The invoice's derived figures after it. pub document: WriteOffDocumentOut, } + +// P6: evidence (plan/served.md P6: extraction, dedup, annotations) +// --------------------------------------------------------------------------- + +/// One annotation's fields, as a human reads them off a file (plan P6 step +/// 5). Every field is optional; the amount is an exact decimal string in +/// the books' own currency. +#[derive(Debug, Default, Clone, Deserialize, JsonSchema)] +pub struct AnnotationFieldsParam { + /// The number printed on the receipt. Engine numbers run from the + /// highest one annotated (`next_receipt_number` in the answer). + pub receipt_number: Option, + /// Who was paid. + pub payee: Option, + /// The amount printed on the receipt, a decimal string such as + /// "12.34". + pub amount: Option, + /// The date printed on the receipt, YYYY-MM-DD. + pub date: Option, + /// How it was paid: a payment method's id or label. + pub instrument: Option, + /// What kind of record it is (the document categories). + pub category: Option, + /// Where it came from. + pub source: Option, + /// Who is to be reimbursed for it, if anyone. + pub reimbursable_to: Option, +} + +/// One recorded annotation, as read back. +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct AnnotationOut { + /// The number printed on the receipt, if recorded. + #[serde(skip_serializing_if = "Option::is_none")] + pub receipt_number: Option, + /// Who was paid, if recorded. + #[serde(skip_serializing_if = "Option::is_none")] + pub payee: Option, + /// The amount printed on the receipt, decimal string, if recorded. + #[serde(skip_serializing_if = "Option::is_none")] + pub amount: Option, + /// The date printed on the receipt, if recorded. + #[serde(skip_serializing_if = "Option::is_none")] + pub date: Option, + /// How it was paid, if recorded. + #[serde(skip_serializing_if = "Option::is_none")] + pub instrument: Option, + /// What kind of record it is, if recorded. + #[serde(skip_serializing_if = "Option::is_none")] + pub category: Option, + /// Where it came from, if recorded. + #[serde(skip_serializing_if = "Option::is_none")] + pub source: Option, + /// Who is to be reimbursed for it, if recorded. + #[serde(skip_serializing_if = "Option::is_none")] + pub reimbursable_to: Option, + /// A free note, if recorded. + #[serde(skip_serializing_if = "Option::is_none")] + pub note: Option, +} + +/// Parameters for `books.find_attachment`: exactly one of its three. +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct FindAttachmentParams { + /// The exact SHA-256 of the file's bytes. + pub sha256: Option, + /// The receipt number recorded on it (books.annotate_attachment). + pub receipt_number: Option, + /// Text it says, case-insensitively (its extracted text; see + /// books.get_receipt view text). + pub text: Option, +} + +/// Result of `books.find_attachment`. +#[derive(Debug, Serialize, JsonSchema)] +pub struct FindAttachmentOut { + /// Every match, withdrawn ones included (marked), oldest first. + pub attachments: Vec, +} + +/// Parameters for `books.annotate_attachment`. +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct AnnotateAttachmentParams { + /// The attachment. + pub attachment_id: u64, + /// What was read off it (the closed field list). + pub fields: AnnotationFieldsParam, + /// A free note. + pub note: Option, +} + +/// Result of `books.annotate_attachment`. +#[derive(Debug, Serialize, JsonSchema)] +pub struct AnnotateAttachmentOut { + /// The attachment. + pub attachment_id: u64, + /// The annotation as stored (it replaces an earlier one). + pub fields: AnnotationOut, + /// The next receipt number the books would give: one past the highest + /// annotated. + pub next_receipt_number: u32, +} + +/// Parameters for `books.set_entry_settles`. +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SetEntrySettlesParams { + /// The entry that pays, such as a reimbursement to a member. + pub entry_id: u64, + /// The entries it settles (what it pays back). Empty clears the record. + pub settles: Vec, +} + +/// Result of `books.set_entry_settles`. +#[derive(Debug, Serialize, JsonSchema)] +pub struct SetEntrySettlesOut { + /// True when recorded. + pub ok: bool, + /// The entry that pays. + pub entry_id: u64, + /// The entries it settles now. + pub settles: Vec, +} + +/// One set of attachments holding the same bytes (plan P6 step 4). +#[derive(Debug, Serialize, JsonSchema)] +pub struct DuplicateGroupOut { + /// The SHA-256 they share. + pub sha256: String, + /// The attachment kept (the oldest). + pub keep: u64, + /// The attachments withdrawn as duplicates of it. + pub withdraw: Vec, + /// The subjects the kept file is linked to on their behalf. + pub links: Vec, +} + +/// Parameters for `books.consolidate_duplicates`. +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ConsolidateDuplicatesParams { + /// True: consolidate. Without it, a preview that changes nothing. + pub confirmed: Option, +} + +/// Result of `books.consolidate_duplicates`. +#[derive(Debug, Serialize, JsonSchema)] +pub struct ConsolidateDuplicatesOut { + /// "preview" or "consolidated". + pub status: String, + /// Each set of attachments holding one file's bytes. + pub groups: Vec, + /// How many attachments were (or would be) withdrawn. + pub withdrawn: usize, +} diff --git a/crates/ledger-package/src/attention.rs b/crates/ledger-package/src/attention.rs index edd697a..ed6a9b5 100644 --- a/crates/ledger-package/src/attention.rs +++ b/crates/ledger-package/src/attention.rs @@ -73,7 +73,12 @@ //! books.set_entry_party); each warning of the fiscal year's //! statement of cash flows (rules.md B-14 rule 12: equipment sold for //! an IOU, Todo; bought on an operating payable, Info; one per -//! entry, its detail the warning's fix). +//! entry, its detail the warning's fix); a filed PDF whose text +//! extraction failed (Todo, one per file: a human must read its +//! pages); a posted entry a member paid personally (a credit on a +//! "due to"/"owed to" liability) that no entry records settling +//! (Todo, one per entry; `entry.settles` only, no FIFO guess, +//! IZ-18). //! 3. Nothing else: books with none of those yield no items, except //! that a figure behind an item that can't be computed (an //! amortization plan past chrono's last day, an aging past exact @@ -862,6 +867,62 @@ pub fn find_with( )), }); } + // A member (or lender) paid for the company and nothing records what + // paid them back: `entry.settles` only, never a FIFO guess (IZ-18). + let owed: Vec = ledger + .chart() + .all() + .filter(|a| { + a.account_type == AccountType::Liability && { + let n = a.name.to_lowercase(); + n.contains("due to") || n.contains("owed to") + } + }) + .map(|a| a.id) + .collect(); + for e in ledger.posted_entries() { + if ledger.settles_entry(e.id()) { + continue; + } + let Some(line) = e + .lines() + .iter() + .find(|l| l.side == ledger_core::Side::Credit && owed.contains(&l.account)) + else { + continue; + }; + // A member paying money INTO the company (a loan or a contribution: + // every debit is a cash account) is not an expense to be paid back. + if !e + .lines() + .iter() + .any(|l| l.side == ledger_core::Side::Debit && !money_cash.contains(&l.account)) + { + continue; + } + let account = account_label(ledger, line.account); + out.push(Attention { + severity: Severity::Todo, + kind: "member_paid", + title: format!("No settlement recorded: {}", e.description()), + detail: format!( + "#{} · {} · {} paid through {}. No payment records settling it, so the books can't say whether the member was paid back.", + e.id(), + date(e.date()), + dollars(line.amount.amount), + account + ), + target: Target::Entry(e.id()), + prompt: format!( + "In wanderbooks, entry {} (\"{}\", {}) was paid by a member. Ask me which payment settled it (a reimbursement, a distribution, or a contribution of the expense) and record it with books.set_entry_settles on that payment, naming {} as settled.", + e.id(), + e.description(), + date(e.date()), + e.id() + ), + tool: Some(("books.set_entry_settles".into(), Value::Null)), + }); + } for a in ledger.chart().all().filter(|a| { a.account_type == AccountType::Liability && { @@ -1203,6 +1264,49 @@ pub fn find_with( tool: Some(("books.verify".into(), Value::Null)), }); } + // A filed PDF whose text couldn't be read: a human must read its + // pages (plan P6 step 7; D-15: no OCR). + let extractions = books.extractions().unwrap_or_default(); + let mut latest: HashMap = HashMap::new(); + for x in &extractions { + let cur = latest.get(&x.attachment_id); + if cur.is_none_or(|c| c.seq < x.seq) { + latest.insert(x.attachment_id, x); + } + } + for a in books.all_attachments().unwrap_or_default() { + if a.withdrawn_at.is_some() { + continue; + } + let is_pdf = + a.mime_type == "application/pdf" || a.file_name.to_lowercase().ends_with(".pdf"); + if !is_pdf + || !latest + .get(&a.id) + .is_some_and(|x| x.status == ledger_store::ExtractionStatus::Failed) + { + continue; + } + out.push(Attention { + severity: Severity::Todo, + kind: "extraction_failed", + title: format!("Couldn't read the text: {}", a.file_name), + detail: format!( + "Attachment {} ({}): reading its text failed, so its words are not in the books. A human must read its pages.", + a.id, a.category + ), + target: Target::Documents, + prompt: format!( + "In wanderbooks, reading attachment {}'s text failed. Show me its pages (books.get_receipt, view page_image, page 1) and let's read what it says; if the pages are pictures only, that is expected and no text is needed.", + a.id + ), + tool: Some(( + "books.get_receipt".into(), + serde_json::json!({"attachment_id": a.id, "view": "page_image", "page": 1}), + )), + }); + } + // The statement of cash flows' warnings this fiscal year (rules.md // B-14 rule 12, D-35): equipment sold for an IOU, flagged with its fix // (Todo); equipment bought on an operating payable, following how the diff --git a/crates/ledger-package/src/books.rs b/crates/ledger-package/src/books.rs index deb0d4b..d35e7bf 100644 --- a/crates/ledger-package/src/books.rs +++ b/crates/ledger-package/src/books.rs @@ -20,8 +20,8 @@ use ledger_core::Ledger; use ledger_store::{ - Attachment, DiskCheck, JournalLine, Reconciliation, Snapshot, Statement, Store, StoreError, - Subject, + Attachment, DiskCheck, Extraction, JournalLine, Reconciliation, Snapshot, Statement, Store, + StoreError, Subject, }; /// The books, read-only, as the package reads them. @@ -47,6 +47,10 @@ pub trait PackageBooks { /// included, in id order. fn all_attachments(&self) -> Result, StoreError>; + /// Every extraction record, in seq order (plan P6): what reading each + /// file found. + fn extractions(&self) -> Result, StoreError>; + /// The subjects attachment `attachment_id` is linked to beyond the /// entry it is filed on (an implementation may list that entry too: /// the package takes the union). @@ -88,6 +92,10 @@ impl PackageBooks for Store { Store::list_attachments(self, None) } + fn extractions(&self) -> Result, StoreError> { + Store::extractions(self) + } + fn links_of(&self, attachment_id: u64) -> Result, StoreError> { Store::attachment_links(self, attachment_id) } @@ -133,6 +141,10 @@ impl PackageBooks for Snapshot { Snapshot::attachments(self, None) } + fn extractions(&self) -> Result, StoreError> { + Snapshot::extractions(self) + } + fn links_of(&self, attachment_id: u64) -> Result, StoreError> { Ok(Snapshot::attachment_links(self)? .into_iter() diff --git a/crates/ledger-package/tests/package.rs b/crates/ledger-package/tests/package.rs index 78d85ec..53c36f7 100644 --- a/crates/ledger-package/tests/package.rs +++ b/crates/ledger-package/tests/package.rs @@ -1272,6 +1272,9 @@ impl PackageBooks for AtSeq<'_> { fn all_attachments(&self) -> Result, ledger_store::StoreError> { PackageBooks::all_attachments(self.0) } + fn extractions(&self) -> Result, ledger_store::StoreError> { + PackageBooks::extractions(self.0) + } fn links_of(&self, id: u64) -> Result, ledger_store::StoreError> { PackageBooks::links_of(self.0, id) } diff --git a/crates/ledger-store/Cargo.toml b/crates/ledger-store/Cargo.toml index 7d571a0..90887ea 100644 --- a/crates/ledger-store/Cargo.toml +++ b/crates/ledger-store/Cargo.toml @@ -8,6 +8,11 @@ description = "Durable persistence for wanderbooks: a hash-chained JSONL journal [dependencies] ledger-core = { path = "../ledger-core" } +# The one extractor (plan P6 step 1, D-53): the writer journals a file's +# text at attach time and on `wanderbooks books extract`, so the extractor +# must be callable here rather than only from the server. It is pure and +# deterministic (no I/O, no network, no time source). +ledger-import = { path = "../ledger-import" } rusqlite = { version = "0.32", features = ["bundled"] } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/crates/ledger-store/src/format.rs b/crates/ledger-store/src/format.rs index 12069c6..372f13d 100644 --- a/crates/ledger-store/src/format.rs +++ b/crates/ledger-store/src/format.rs @@ -63,6 +63,7 @@ pub const KNOWN_FEATURES: &[&str] = &[ DOCUMENT_LINE_PROJECTS, DOCUMENT_STANDING, DOCUMENTS_V2, + ATTACHMENT_TEXT, ]; /// Feature: `period.closed` closes to members with `"rounding": @@ -198,6 +199,15 @@ pub const DOCUMENT_STANDING: &str = "document_standing"; /// discount and the fee, or refuse the unknown values, silently. pub const DOCUMENTS_V2: &str = "documents_v2"; +/// Feature: the evidence pipeline (plan P6): `attachment.text_extracted` +/// (a file's text, journaled as an extraction record), `attachment.annotated` +/// (the facts a human read off a file), `entry.settles` (which entries a +/// payment pays back), and `attachment.withdrawn` carrying `duplicate_of`. +/// An older version doesn't know those event types, or would withdraw a +/// duplicate without saying what it duplicates; it stops at the +/// declaration instead (rules.md B-27, B-44). +pub const ATTACHMENT_TEXT: &str = "attachment_text"; + /// The line type that declares the journal format. pub const FORMAT_SET: &str = "journal.format_set"; @@ -218,7 +228,9 @@ pub const EVENT_TYPES: &[&str] = &[ "asset.serial_set", "asset.tagged", "asset.tax_basis_set", + "attachment.annotated", "attachment.linked", + "attachment.text_extracted", "attachment.unlinked", "attachment.withdrawn", "bill.created", @@ -239,6 +251,7 @@ pub const EVENT_TYPES: &[&str] = &[ "entry.party_set", "entry.posted", "entry.reversed", + "entry.settles", "entry.vendor_set", "fx.rate_set", "invoice.created", @@ -860,6 +873,40 @@ mod tests { assert_eq!(up.adds(), vec!["the journal feature document_standing"]); } + /// The version before P6 (every feature through `documents_v2`) stops + /// at a declaration of `attachment_text` as newer, never dropping a + /// file's extraction, annotations or settlements; and books that + /// declared every earlier feature still wait for it (B-27, D-39). + #[test] + fn attachment_text_is_its_own_feature() { + assert!(KNOWN_FEATURES.contains(&ATTACHMENT_TEXT)); + let before: Vec<&str> = KNOWN_FEATURES + .iter() + .copied() + .filter(|f| *f != ATTACHMENT_TEXT) + .collect(); + let l = line( + FORMAT_SET, + json!({"format": 2, "features": [DOCUMENTS_V2, ATTACHMENT_TEXT]}), + ); + match declared_knowing(&l, &before) { + Err(StoreError::NewerJournal { need, .. }) => { + assert_eq!(need, "journal features attachment_text") + } + other => panic!("expected NewerJournal, got {other:?}"), + } + let older = JournalFormat { + format: 2, + features: before.iter().map(|f| f.to_string()).collect(), + }; + assert!(matches!( + older.require_feature(ATTACHMENT_TEXT), + Err(StoreError::FormatUpgradePending { .. }) + )); + let up = older.upgrade().expect("waits for it"); + assert_eq!(up.adds(), vec!["the journal feature attachment_text"]); + } + /// The version before P4b S4.2 (every feature through /// `document_standing`) stops at a declaration of `documents_v2` as /// newer, never restoring a party's email or default terms as absent diff --git a/crates/ledger-store/src/lib.rs b/crates/ledger-store/src/lib.rs index ee22ef9..a6648f3 100644 --- a/crates/ledger-store/src/lib.rs +++ b/crates/ledger-store/src/lib.rs @@ -47,8 +47,8 @@ pub use format::{FormatUpgrade, JournalFormat, SUPPORTED_JOURNAL_FORMAT}; pub use journal::{GENESIS_PREV, JournalLine, JournalWriter}; pub use snapshot::{DiskCheck, Snapshot}; pub use store::{ - Attachment, BillPreview, DepreciationRun, DisposalPreview, EntryRow, EventRow, InvoicePreview, - JOURNAL_FILE, LEGACY_DB_FILE, LOCK_FILE, MAX_RECURRING_RUN, PayrollLineInput, PreviewPosting, - Reconciliation, Statement, Store, Subject, VerifyReport, WriteOffEvidence, database_in_use, - db_file, db_file_for, is_database_file, + Attachment, BillPreview, DepreciationRun, DisposalPreview, EntryRow, EventRow, Extraction, + ExtractionStatus, InvoicePreview, JOURNAL_FILE, LEGACY_DB_FILE, LOCK_FILE, MAX_RECURRING_RUN, + PayrollLineInput, PreviewPosting, Reconciliation, Statement, Store, Subject, VerifyReport, + WriteOffEvidence, database_in_use, db_file, db_file_for, is_database_file, }; diff --git a/crates/ledger-store/src/meta.rs b/crates/ledger-store/src/meta.rs index 06a5005..42e1b1f 100644 --- a/crates/ledger-store/src/meta.rs +++ b/crates/ledger-store/src/meta.rs @@ -19,7 +19,9 @@ use chrono::NaiveDate; use ledger_core::arap::{SendKind, Sent, SentVia}; use ledger_core::conventions::{Approval, ConventionKind, NewConvention, NewDecision, NewOpenItem}; use ledger_core::settings::{Change, Source}; -use ledger_core::{ClassId, ProjectFacts, ProjectKind, ProjectStatus}; +use ledger_core::{ + Annotation, AnnotationFields, ClassId, ProjectFacts, ProjectKind, ProjectStatus, +}; use serde_json::{Map, Value, json}; /// The journal format these events need: an older reader stops at the @@ -466,6 +468,162 @@ pub fn invoice_written_off_of(p: &Value) -> Result { }) } +/// `attachment.text_extracted {attachment_id, text_sha256, extractor, +/// version, pages, status}` (plan P6, feature `attachment_text`). +pub fn text_extracted( + attachment_id: u64, + text_sha256: Option<&str>, + extractor: &str, + version: &str, + pages: u32, + status: &str, +) -> Value { + json!({ + "attachment_id": attachment_id, + "text_sha256": text_sha256, + "extractor": extractor, + "version": version, + "pages": pages, + "status": status, + }) +} + +/// Read an `attachment.text_extracted`: `(attachment_id, text_sha256, +/// extractor, version, pages, status)`. A status this version doesn't know +/// is newer (B-27: never "corrupt" for newer). +pub fn text_extracted_of(line: &crate::journal::JournalLine) -> Result { + const E: &str = "attachment.text_extracted"; + let p = &line.payload; + let status = str_of(p, E, "status")?; + if !matches!(status.as_str(), "ok" | "none" | "failed") { + return Err(crate::format::newer_content( + line, + format!("the extraction status {status:?}"), + )); + } + Ok(( + u64_of(p, E, "attachment_id")?, + opt_str(p, E, "text_sha256")?, + str_of(p, E, "extractor")?, + str_of(p, E, "version")?, + u32::try_from(u64_of(p, E, "pages")?).map_err(|_| corrupt(E, "pages"))?, + status, + )) +} + +/// What an `attachment.text_extracted` says: attachment id, text blob, +/// extractor, version, pages, status. +pub type TextExtracted = (u64, Option, String, String, u32, String); + +/// The annotation fields this version knows (the closed list, plan P6 +/// step 5). +const ANNOTATION_FIELDS: &[&str] = &[ + "receipt_number", + "payee", + "amount", + "date", + "instrument", + "category", + "source", + "reimbursable_to", +]; + +/// `attachment.annotated {attachment_id, fields{...}, note?}`: only the +/// fields given are written, so a canonical payload. +pub fn annotated(a: &Annotation) -> Value { + let mut fields = Map::new(); + put(&mut fields, "receipt_number", a.fields.receipt_number); + put(&mut fields, "payee", a.fields.payee.clone()); + put( + &mut fields, + "amount", + a.fields.amount.map(|v| v.to_string()), + ); + put(&mut fields, "date", a.fields.date.map(|d| d.to_string())); + put(&mut fields, "instrument", a.fields.instrument.clone()); + put(&mut fields, "category", a.fields.category.clone()); + put(&mut fields, "source", a.fields.source.clone()); + put( + &mut fields, + "reimbursable_to", + a.fields.reimbursable_to.clone(), + ); + let mut o = Map::new(); + o.insert("attachment_id".into(), json!(a.attachment_id)); + o.insert("fields".into(), Value::Object(fields)); + put(&mut o, "note", a.note.clone()); + Value::Object(o) +} + +/// Read an `attachment.annotated`. A field this version doesn't know, or a +/// mistyped one, is judged by its kind: unknown is newer (a later version +/// wrote it, B-27), mistyped is corrupt. +pub fn annotated_of(line: &crate::journal::JournalLine) -> Result { + const E: &str = "attachment.annotated"; + let p = &line.payload; + let fields = p + .get("fields") + .filter(|v| v.is_object()) + .ok_or_else(|| corrupt(E, "fields"))?; + for key in fields.as_object().expect("object").keys() { + if !ANNOTATION_FIELDS.contains(&key.as_str()) { + return Err(crate::format::newer_content( + line, + format!("the annotation field {key:?}"), + )); + } + } + let text = |k: &str| opt_str(fields, E, k); + let receipt_number = match fields.get("receipt_number") { + None | Some(Value::Null) => None, + Some(v) => Some( + v.as_u64() + .and_then(|n| u32::try_from(n).ok()) + .ok_or_else(|| corrupt(E, "receipt_number"))?, + ), + }; + let amount = match text("amount")? { + None => None, + Some(s) => Some( + s.parse() + .map_err(|_| StoreError::JournalCorrupt(format!("{E}: amount")))?, + ), + }; + let date = opt_date(fields, E, "date")?; + Ok(Annotation { + attachment_id: u64_of(p, E, "attachment_id")?, + fields: AnnotationFields { + receipt_number, + payee: text("payee")?, + amount, + date, + instrument: text("instrument")?, + category: text("category")?, + source: text("source")?, + reimbursable_to: text("reimbursable_to")?, + }, + note: opt_str(p, E, "note")?, + }) +} + +/// `entry.settles {entry_id, settles: [entry_id]}` (plan P6). +pub fn entry_settles(entry: u64, settles: &[u64]) -> Value { + json!({ "entry_id": entry, "settles": settles }) +} + +/// Read an `entry.settles`: `(entry, settles)`. +pub fn entry_settles_of(p: &Value) -> Result<(u64, Vec), StoreError> { + const E: &str = "entry.settles"; + let settles = p + .get("settles") + .and_then(Value::as_array) + .ok_or_else(|| corrupt(E, "settles"))? + .iter() + .map(|v| v.as_u64().ok_or_else(|| corrupt(E, "settles"))) + .collect::, _>>()?; + Ok((u64_of(p, E, "entry_id")?, settles)) +} + #[cfg(test)] mod tests { use super::*; @@ -549,6 +707,50 @@ mod tests { entry_approved_of(&entry_approved(5, &a), 9).unwrap(), (5, a) ); + let line = crate::journal::JournalLine { + seq: 11, + ts: "2026-10-27T00:00:00Z".into(), + line_type: "attachment.annotated".into(), + payload: annotated(&Annotation { + attachment_id: 3, + fields: AnnotationFields { + receipt_number: Some(7), + payee: Some("Harbor Design".into()), + amount: Some("12.34".parse().unwrap()), + date: NaiveDate::from_ymd_opt(2026, 3, 1), + instrument: Some("Check 1234".into()), + category: None, + source: Some("a folder".into()), + reimbursable_to: Some("Avery Quill".into()), + }, + note: Some("read on 2026-03-02".into()), + }), + prev: String::new(), + }; + let read = annotated_of(&line).unwrap(); + assert_eq!(annotated(&read), line.payload); + assert_eq!( + entry_settles_of(&entry_settles(4, &[1, 2])).unwrap(), + (4, vec![1, 2]) + ); + let x = crate::journal::JournalLine { + seq: 12, + ts: "2026-10-27T00:00:00Z".into(), + line_type: "attachment.text_extracted".into(), + payload: text_extracted(3, Some("00ff"), "wanderbooks", "1", 2, "ok"), + prev: String::new(), + }; + assert_eq!( + text_extracted_of(&x).unwrap(), + ( + 3, + Some("00ff".into()), + "wanderbooks".into(), + "1".into(), + 2, + "ok".into() + ) + ); } /// INVARIANT 1. diff --git a/crates/ledger-store/src/schema.rs b/crates/ledger-store/src/schema.rs index c90c065..2d570cd 100644 --- a/crates/ledger-store/src/schema.rs +++ b/crates/ledger-store/src/schema.rs @@ -5,7 +5,7 @@ //! is the current version. /// Current schema version. -pub const SCHEMA_VERSION: u32 = 17; +pub const SCHEMA_VERSION: u32 = 18; /// Migrations, in order. `MIGRATIONS[i]` migrates from version `i` to `i+1`. pub const MIGRATIONS: &[&str] = &[ @@ -434,5 +434,39 @@ CREATE TABLE IF NOT EXISTS invoice_sends( reference TEXT NULL, note TEXT NULL ); +"#, + r#" +-- v18: evidence (plan P6, feature attachment_text): what was read out of +-- a file (`attachment.text_extracted`), the facts a human annotated it +-- with (`attachment.annotated`), and the duplicate an attachment was +-- withdrawn as. Extraction is its own record per (attachment, extractor, +-- version): an extractor upgrade adds one, never replaces one. A text +-- hash is the extraction's blob; the text itself is content-addressed in +-- the blob store. Amounts stay TEXT (exact decimal), never REAL. +CREATE TABLE IF NOT EXISTS extractions( + attachment_id INTEGER NOT NULL, + extractor TEXT NOT NULL, + version TEXT NOT NULL, + text_sha256 TEXT NULL, + pages INTEGER NOT NULL, + status TEXT NOT NULL, + seq INTEGER NOT NULL, + PRIMARY KEY(attachment_id, extractor, version) +); +CREATE INDEX IF NOT EXISTS idx_extractions_text ON extractions(text_sha256); +CREATE TABLE IF NOT EXISTS annotations( + attachment_id INTEGER PRIMARY KEY, + receipt_number INTEGER NULL, + payee TEXT NULL, + amount TEXT NULL, + date TEXT NULL, + instrument TEXT NULL, + category TEXT NULL, + source TEXT NULL, + reimbursable_to TEXT NULL, + note TEXT NULL, + seq INTEGER NOT NULL +); +ALTER TABLE attachments ADD COLUMN duplicate_of INTEGER NULL; "#, ]; diff --git a/crates/ledger-store/src/snapshot.rs b/crates/ledger-store/src/snapshot.rs index c98e502..2c977a5 100644 --- a/crates/ledger-store/src/snapshot.rs +++ b/crates/ledger-store/src/snapshot.rs @@ -19,10 +19,10 @@ use crate::error::StoreError; use crate::format::JournalFormat; use crate::journal::{GENESIS_PREV, JournalLine}; use crate::store::{ - Attachment, BLOBS_DIR, JOURNAL_FILE, Reconciliation, Statement, Store, Subject, - apply_line_to_sqlite, compare_projections, projection_rows, query_attachment_links, - query_attachments, query_reconciliations, query_statements, read_attachment_bytes, - replay_line_into_ledger, + Attachment, BLOBS_DIR, Extraction, JOURNAL_FILE, Reconciliation, Statement, Store, Subject, + apply_line_to_sqlite, compare_projections, projection_rows, query_annotations, + query_attachment_links, query_attachments, query_extractions, query_reconciliations, + query_statements, read_attachment_bytes, replay_line_into_ledger, }; use ledger_core::AccountId; use ledger_core::{Currency, EntryId, Ledger}; @@ -261,6 +261,16 @@ impl Snapshot { query_attachments(&self.conn, entry_id) } + /// Every extraction record, in seq order (as [`Store::extractions`]). + pub fn extractions(&self) -> Result, StoreError> { + query_extractions(&self.conn) + } + + /// Every annotation, by attachment id (as [`Store::annotations`]). + pub fn annotations(&self) -> Result, StoreError> { + query_annotations(&self.conn) + } + /// Every attachment link beyond a receipt's filing entry, as /// (attachment id, subject), by attachment then subject: what /// [`Store::attachment_links`] reads, for every attachment at once. diff --git a/crates/ledger-store/src/store.rs b/crates/ledger-store/src/store.rs index 479181d..e7e1565 100644 --- a/crates/ledger-store/src/store.rs +++ b/crates/ledger-store/src/store.rs @@ -45,6 +45,7 @@ use ledger_core::{ payroll::{PayrollAccounts, payroll_entry_plan, payroll_remit_plan}, reports, }; +use ledger_import::extract::{EXTRACTOR as E_EXTRACTOR, VERSION as E_VERSION}; use rusqlite::{Connection, OptionalExtension, params}; use rust_decimal::Decimal; use serde_json::{Value, json}; @@ -299,7 +300,11 @@ pub struct Attachment { pub blob_sha: String, /// Linked entry, if any. pub entry_id: Option, - /// `receipts`, `formation`, `legal`, `tax`, `accounting`, `other`. + /// `receipts`, or a document's: `formation`, `legal`, `tax`, + /// `accounting`, `statement` (a bank or card statement), `contract`, + /// `photo`, `warranty`, `import_source` (a bank export), `notices`, + /// `correspondence`, `plans`, `history`, `other` (D-20: every company + /// record, not only what the CPA needs). pub category: String, /// Original file name. pub file_name: String, @@ -315,6 +320,64 @@ pub struct Attachment { pub withdrawn_at: Option, /// Why it was withdrawn, if it was. pub withdrawn_reason: Option, + /// The attachment it duplicates, when it was withdrawn as one + /// (`attachment.withdrawn`'s `duplicate_of`, plan P6 step 4). + pub duplicate_of: Option, +} + +/// What reading a file's text found (`attachment.text_extracted`, plan P6 +/// step 2). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ExtractionStatus { + /// Text was read. + Ok, + /// There is no text to read (an image, an empty sheet): D-15's page + /// pictures, not OCR. + None, + /// Reading was tried and failed: a human must read the file. + Failed, +} + +impl ExtractionStatus { + /// Its wire name, as journaled. + pub fn key(self) -> &'static str { + match self { + ExtractionStatus::Ok => "ok", + ExtractionStatus::None => "none", + ExtractionStatus::Failed => "failed", + } + } + + /// A wire name back as a status. + pub fn parse(key: &str) -> Option { + match key { + "ok" => Some(ExtractionStatus::Ok), + "none" => Some(ExtractionStatus::None), + "failed" => Some(ExtractionStatus::Failed), + _ => None, + } + } +} + +/// One record of reading a file's text, per (attachment, extractor, +/// version): an extractor upgrade journals a new record, never replaces +/// one. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Extraction { + /// The attachment read. + pub attachment_id: u64, + /// The text's blob (its SHA-256), when there is text. + pub text_sha256: Option, + /// What read it (`ledger_import::extract::EXTRACTOR`). + pub extractor: String, + /// That extractor's version. + pub version: String, + /// How many pages the file has (0 when it has none). + pub pages: u32, + /// What was found. + pub status: ExtractionStatus, + /// The journal seq of the record (the latest wins when reading). + pub seq: u64, } /// Input to [`Store::run_payroll`]: one employee's gross pay and every @@ -3849,7 +3912,13 @@ impl Store { return Err(StoreError::WriteOffNeedsEvidence); }; // Every check on the evidence before anything is journaled: a bad - // attachment refuses the whole write-off. + // attachment refuses the whole write-off. A file whose bytes are + // filed already is that record, linked to the write-off's entry and + // never a second one (plan P6 step 4, `Self::attach`); it is checked + // here as an existing attachment is, except that a link it already + // has to this invoice (the same letter evidencing a second write-off + // of it) is kept, not refused (found 2026-10-28, merging S7 and P6: + // the refusal came after the entry was journaled). if let WriteOffEvidence::Existing(id) = evidence { self.check_linkable(id)?; if self @@ -3862,6 +3931,11 @@ impl Store { )); } } + if let WriteOffEvidence::File { bytes, .. } = evidence + && let Some(existing) = self.live_attachment_with_sha(&sha256_hex(bytes))? + { + self.check_linkable(existing.id)?; + } // The bytes hit the content-addressed store before anything is // journaled: an orphan blob from a later refusal is harmless // (unreferenced, deduplicated), a journal line naming a blob that @@ -3904,7 +3978,12 @@ impl Store { id } }; - self.link_attachment(attachment_id, Subject::Invoice(invoice))?; + if !self + .attachment_links(attachment_id)? + .contains(&Subject::Invoice(invoice)) + { + self.link_attachment(attachment_id, Subject::Invoice(invoice))?; + } Ok((entry_id, attachment_id)) } /// The journal features a settlement needs (rules.md B-27, B-42): @@ -4459,12 +4538,32 @@ impl Store { } /// One filing of bytes into the blob store and journal. + /// + /// Plan P6 step 4 (defect 4): bytes already filed (and not withdrawn) + /// are one file, not a second record: the existing attachment is + /// linked to the new subject and returned, `deduplicated` true. A + /// filing with no subject (an unlinked receipt, a document) returns + /// the existing record too, for the caller to link later. Legacy + /// duplicates stay as they were written. fn attach(&mut self, req: AttachRequest<'_>) -> Result<(u64, String, bool), StoreError> { if let Some(eid) = req.entry_id { self.ledger .get_entry(eid) .ok_or(LedgerError::UnknownEntry(eid))?; } + let sha = sha256_hex(req.bytes); + if let Some(existing) = self.live_attachment_with_sha(&sha)? { + if let Some(e) = req.entry_id { + let covered = existing.entry_id == Some(e) + || self + .attachment_links(existing.id)? + .contains(&Subject::Entry(e)); + if !covered { + self.link_attachment(existing.id, Subject::Entry(e))?; + } + } + return Ok((existing.id, sha, true)); + } // Bytes hit the content-addressed store first; an orphan blob from a // failed journal append is harmless (unreferenced, deduplicated). let (sha, dedup) = self.blobs.put(req.bytes)?; @@ -4487,9 +4586,212 @@ impl Store { }), )?; apply_line_to_sqlite(&self.conn, &line)?; + // Plan P6 step 2: a file is read as it is filed. Books that haven't + // declared `attachment_text` yet keep filing (the owner's go-ahead + // gates the extraction records, not attaching a receipt); their + // old and new files are read by `wanderbooks books extract` after + // the upgrade. + if self.format.has_feature(crate::format::ATTACHMENT_TEXT) { + self.extract_and_journal(id, req.bytes, req.mime_type, req.file_name)?; + } Ok((id, sha, dedup)) } + /// The first attachment holding `sha`'s bytes that is not withdrawn. + fn live_attachment_with_sha(&self, sha: &str) -> Result, StoreError> { + Ok(query_attachments(&self.conn, None)? + .into_iter() + .find(|a| a.blob_sha.eq_ignore_ascii_case(sha) && a.withdrawn_at.is_none())) + } + + /// Read a new attachment's text and journal the record (plan P6 step + /// 2): a content-addressed text blob, the extractor and its version, + /// the pages, and `ok`, `none` or `failed`. A file with no text still + /// gets its `none` record: the books say reading was tried. + fn extract_and_journal( + &mut self, + attachment_id: u64, + bytes: &[u8], + mime_type: &str, + file_name: &str, + ) -> Result { + if let Some(already) = self.extraction_for(attachment_id, E_EXTRACTOR, E_VERSION)? { + return Ok(already); + } + let got = ledger_import::extract::extract(bytes, mime_type, file_name); + let text_sha = match &got.text { + Some(text) => Some(self.blobs.put(text.as_bytes())?.0), + None => None, + }; + let line = self.append_journal( + "attachment.text_extracted", + crate::meta::text_extracted( + attachment_id, + text_sha.as_deref(), + E_EXTRACTOR, + E_VERSION, + got.pages, + got.status.key(), + ), + )?; + apply_line_to_sqlite(&self.conn, &line)?; + Ok(Extraction { + attachment_id, + text_sha256: text_sha, + extractor: E_EXTRACTOR.into(), + version: E_VERSION.into(), + pages: got.pages, + status: match got.status { + ledger_import::extract::ExtractStatus::Ok => ExtractionStatus::Ok, + ledger_import::extract::ExtractStatus::None => ExtractionStatus::None, + ledger_import::extract::ExtractStatus::Failed => ExtractionStatus::Failed, + }, + seq: line.seq, + }) + } + + /// Read an attachment's text now and journal it (plan P6 step 2; + /// idempotent on attachment, extractor and version). A refusal (an + /// unknown attachment, books that haven't declared `attachment_text`) + /// journals nothing. The backfill + /// (`wanderbooks books extract ID`) loops this over old attachments, + /// one at a time. + pub fn extract_attachment(&mut self, attachment_id: u64) -> Result { + let a = self + .attachment_by_id(attachment_id)? + .ok_or(StoreError::UnknownAttachment(attachment_id))?; + if let Some(already) = self.extraction_for(attachment_id, E_EXTRACTOR, E_VERSION)? { + return Ok(already); + } + self.ensure_feature(crate::format::ATTACHMENT_TEXT)?; + let bytes = self.blobs.get(&a.blob_sha)?; + self.extract_and_journal(attachment_id, &bytes, &a.mime_type, &a.file_name) + } + + /// The newest extraction record for an attachment, if any. + pub fn extraction(&self, attachment_id: u64) -> Result, StoreError> { + Ok(query_extractions(&self.conn)? + .into_iter() + .filter(|x| x.attachment_id == attachment_id) + .max_by_key(|x| x.seq)) + } + + /// The extraction record for exactly this extractor and version. + fn extraction_for( + &self, + attachment_id: u64, + extractor: &str, + version: &str, + ) -> Result, StoreError> { + Ok(query_extractions(&self.conn)?.into_iter().find(|x| { + x.attachment_id == attachment_id && x.extractor == extractor && x.version == version + })) + } + + /// The text an attachment's newest extraction read, if any. + pub fn extraction_text(&self, attachment_id: u64) -> Result, StoreError> { + let Some(x) = self.extraction(attachment_id)? else { + return Ok(None); + }; + let Some(sha) = x.text_sha256 else { + return Ok(None); + }; + Ok(Some( + String::from_utf8_lossy(&self.blobs.get(&sha)?).into_owned(), + )) + } + + /// Every extraction record, in seq order. + pub fn extractions(&self) -> Result, StoreError> { + query_extractions(&self.conn) + } + + /// Every annotation, by attachment id (plan P6 step 5). + pub fn annotations(&self) -> Result, StoreError> { + query_annotations(&self.conn) + } + + /// An attachment by id, withdrawn or not. + pub fn attachment_by_id(&self, id: u64) -> Result, StoreError> { + Ok(query_attachments(&self.conn, None)? + .into_iter() + .find(|a| a.id == id)) + } + + /// Attachments whose bytes hold `text`, case-insensitively: the text + /// search of `books.find_attachment` (plan P6 step 2's extraction is + /// what makes a receipt searchable by what it says). + pub fn attachments_holding_text(&self, text: &str) -> Result, StoreError> { + let needle = text.to_lowercase(); + let mut out = Vec::new(); + for a in query_attachments(&self.conn, None)? { + if self + .extraction_text(a.id)? + .is_some_and(|t| t.to_lowercase().contains(&needle)) + { + out.push(a); + } + } + Ok(out) + } + + /// Attachments annotated with `receipt_number`. + pub fn attachments_with_receipt_number( + &self, + receipt_number: u32, + ) -> Result, StoreError> { + let ids: Vec = self + .conn + .prepare("SELECT attachment_id FROM annotations WHERE receipt_number = ?")? + .query_map(params![i64::from(receipt_number)], |r| r.get(0))? + .collect::>()?; + Ok(query_attachments(&self.conn, None)? + .into_iter() + .filter(|a| ids.contains(&(a.id as i64))) + .collect()) + } + + /// Record what a human read off a file (`attachment.annotated`, plan + /// P6 step 5), replacing an earlier annotation of the same file. The + /// fields are the closed list of [`ledger_core::AnnotationFields`]; + /// a refusal journals nothing. + pub fn annotate_attachment( + &mut self, + annotation: ledger_core::Annotation, + ) -> Result { + self.ensure_feature(crate::format::ATTACHMENT_TEXT)?; + let a = self + .attachment_by_id(annotation.attachment_id)? + .ok_or(StoreError::UnknownAttachment(annotation.attachment_id))?; + if a.withdrawn_at.is_some() { + return Err(StoreError::AttachmentWithdrawn(a.id)); + } + let annotation = annotation.checked()?; + self.rehearse(|l| l.annotate_attachment(annotation.clone()))?; + let line = + self.append_journal("attachment.annotated", crate::meta::annotated(&annotation))?; + apply_line_to_sqlite(&self.conn, &line)?; + self.ledger.annotate_attachment(annotation.clone())?; + Ok(annotation) + } + + /// Record which entries an entry settles (`entry.settles`, plan P6 + /// step 5): what a reimbursement names, and the only source of a + /// member-paid entry's "reimbursed" (IZ-18). + pub fn set_entry_settles( + &mut self, + entry: EntryId, + settles: &[EntryId], + ) -> Result<(), StoreError> { + self.ensure_feature(crate::format::ATTACHMENT_TEXT)?; + self.rehearse(|l| l.set_entry_settles(entry, settles.to_vec()))?; + let line = + self.append_journal("entry.settles", crate::meta::entry_settles(entry, settles))?; + apply_line_to_sqlite(&self.conn, &line)?; + self.ledger.set_entry_settles(entry, settles.to_vec())?; + Ok(()) + } + /// Register an amortization schedule. Journaled as `schedule.registered`, /// with its `convention` (`daily` or `monthly_even`). A daily schedule /// needs journal feature `schedule_convention` (refused, journaling @@ -5119,6 +5421,32 @@ impl Store { &mut self, attachment_id: u64, reason: &str, + ) -> Result<(), StoreError> { + self.withdraw_attachment_as(attachment_id, reason, None) + } + + /// [`Self::withdraw_attachment`] of a duplicate, naming the attachment + /// it duplicates (`attachment.withdrawn`'s `duplicate_of`, plan P6 + /// step 4). + pub fn withdraw_attachment_duplicate( + &mut self, + attachment_id: u64, + reason: &str, + duplicate_of: u64, + ) -> Result<(), StoreError> { + if duplicate_of == attachment_id { + return Err(StoreError::Ledger(LedgerError::InvalidEvidence( + "an attachment doesn't duplicate itself".into(), + ))); + } + self.withdraw_attachment_as(attachment_id, reason, Some(duplicate_of)) + } + + fn withdraw_attachment_as( + &mut self, + attachment_id: u64, + reason: &str, + duplicate_of: Option, ) -> Result<(), StoreError> { let reason = reason.trim(); if reason.is_empty() { @@ -5128,9 +5456,21 @@ impl Store { if let Some(sub) = self.attachment_links(attachment_id)?.first() { return Err(StoreError::AlreadyLinked(attachment_id, sub.describe())); } + if let Some(dup) = duplicate_of { + // The one it duplicates must be a different attachment, and + // `duplicate_of` is a field only books with the feature read + // (plan P6). + self.ensure_feature(crate::format::ATTACHMENT_TEXT)?; + self.attachment_by_id(dup)? + .ok_or(StoreError::UnknownAttachment(dup))?; + } let line = self.append_journal( "attachment.withdrawn", - json!({ "attachment_id": attachment_id, "reason": reason }), + json!({ + "attachment_id": attachment_id, + "reason": reason, + "duplicate_of": duplicate_of, + }), )?; apply_line_to_sqlite(&self.conn, &line)?; Ok(()) @@ -5789,6 +6129,20 @@ impl Store { "payroll runs diverged between journal, live ledger, and sqlite projection".into(), )); } + // The text an extraction recorded is a stored file too: present, + // and the hash it names (plan P6 step 2). It is never re-derived + // here (an extractor upgrade journals a new record, never replaces + // one). + for x in query_extractions(&self.conn)? { + if let Some(sha) = x.text_sha256 { + self.blobs.get(&sha).map_err(|e| { + StoreError::VerifyMismatch(format!( + "attachment {}'s extracted text doesn't verify: {e}", + x.attachment_id + )) + })?; + } + } // Finally, the whole projection, not just what reports read: every // table, row for row, against a fresh projection of the journal. compare_projections( @@ -5848,7 +6202,9 @@ impl Store { /// The SHA-256 of every stored file the books name (receipts and /// documents, withdrawn ones included). pub fn stored_blob_shas(&self) -> Result, StoreError> { - let mut stmt = self.conn.prepare("SELECT sha256 FROM blobs")?; + let mut stmt = self.conn.prepare( + "SELECT sha256 FROM blobs UNION SELECT text_sha256 FROM extractions WHERE text_sha256 IS NOT NULL", + )?; let shas = stmt .query_map([], |r| r.get::<_, String>(0))? .collect::, _>>()?; @@ -7031,8 +7387,22 @@ fn replay_line( | "receipt.linked" | "receipt.unlinked" | "document.filed" - | "attachment.withdrawn" => { - // Not accounting state; the projection records them. + | "attachment.withdrawn" + | "attachment.text_extracted" => { + // Not accounting state; the projection records them. (The text + // record is checked for shape here: an unknown status is newer, + // rules.md B-27.) + if line.line_type == "attachment.text_extracted" { + crate::meta::text_extracted_of(line)?; + } + } + "attachment.annotated" => { + let a = crate::meta::annotated_of(line)?; + ledger.replay_annotate_attachment(a)?; + } + "entry.settles" => { + let (entry, settles) = crate::meta::entry_settles_of(p)?; + ledger.replay_set_entry_settles(entry, settles)?; } "payroll.taxes_remitted" => { // Names the remittance entry (rebuilt by the entry events just @@ -8135,14 +8505,56 @@ fn apply_line_rows(conn: &Connection, line: &JournalLine) -> Result<(), StoreErr } "attachment.withdrawn" => { conn.execute( - "UPDATE attachments SET withdrawn_at = ?, withdrawn_reason = ? WHERE id = ?", + "UPDATE attachments SET withdrawn_at = ?, withdrawn_reason = ?, duplicate_of = ? WHERE id = ?", params![ line.ts, get_str(p, "reason")?, + p.get("duplicate_of").and_then(|v| v.as_u64()).map(|v| v as i64), get_u64(p, "attachment_id")? as i64, ], )?; } + "attachment.text_extracted" => { + let (id, sha, extractor, version, pages, status) = + crate::meta::text_extracted_of(line)?; + conn.execute( + "INSERT INTO extractions(attachment_id, extractor, version, text_sha256, pages, status, seq) + VALUES (?,?,?,?,?,?,?) + ON CONFLICT(attachment_id, extractor, version) DO UPDATE SET + text_sha256 = excluded.text_sha256, pages = excluded.pages, + status = excluded.status, seq = excluded.seq", + params![id as i64, extractor, version, sha, i64::from(pages), status, line.seq as i64], + )?; + } + "attachment.annotated" => { + let a = crate::meta::annotated_of(line)?; + conn.execute( + "INSERT INTO annotations(attachment_id, receipt_number, payee, amount, date, instrument, category, source, reimbursable_to, note, seq) + VALUES (?,?,?,?,?,?,?,?,?,?,?) + ON CONFLICT(attachment_id) DO UPDATE SET + receipt_number = excluded.receipt_number, payee = excluded.payee, + amount = excluded.amount, date = excluded.date, + instrument = excluded.instrument, category = excluded.category, + source = excluded.source, reimbursable_to = excluded.reimbursable_to, + note = excluded.note, seq = excluded.seq", + params![ + a.attachment_id as i64, + a.fields.receipt_number.map(i64::from), + a.fields.payee, + a.fields.amount.map(|v| v.to_string()), + a.fields.date.map(|d| d.to_string()), + a.fields.instrument, + a.fields.category, + a.fields.source, + a.fields.reimbursable_to, + a.note, + line.seq as i64, + ], + )?; + } + "entry.settles" => { + // Rebuilt into the ledger by replay; no report rows. + } "asset.registered" => { let cost = &p["cost"]; let salvage = &p["salvage_value"]; @@ -8920,7 +9332,8 @@ pub(crate) fn query_attachments( let mut sql = String::from( "SELECT a.id, a.blob_sha, a.entry_id, a.category, \ COALESCE(a.file_name, b.original_name), COALESCE(a.mime_type, b.mime_type), \ - b.size_bytes, a.description, a.created_at, a.withdrawn_at, a.withdrawn_reason \ + b.size_bytes, a.description, a.created_at, a.withdrawn_at, a.withdrawn_reason, \ + a.duplicate_of \ FROM attachments a \ JOIN blobs b ON b.sha256 = a.blob_sha", ); @@ -8971,9 +9384,61 @@ fn row_to_attachment(r: &rusqlite::Row) -> rusqlite::Result { created_at: r.get(8)?, withdrawn_at: r.get(9)?, withdrawn_reason: r.get(10)?, + duplicate_of: r.get::<_, Option>(11)?.map(|v| v as u64), }) } +/// Every extraction record, in seq order (plan P6). +pub(crate) fn query_extractions(conn: &Connection) -> Result, StoreError> { + let mut stmt = conn.prepare( + "SELECT attachment_id, text_sha256, extractor, version, pages, status, seq \ + FROM extractions ORDER BY seq, attachment_id, extractor, version", + )?; + let rows = stmt.query_map([], |r| { + let status: String = r.get(5)?; + Ok(Extraction { + attachment_id: r.get::<_, i64>(0)? as u64, + text_sha256: r.get(1)?, + extractor: r.get(2)?, + version: r.get(3)?, + pages: r.get::<_, i64>(4)? as u32, + status: ExtractionStatus::parse(&status).unwrap_or(ExtractionStatus::Failed), + seq: r.get::<_, i64>(6)? as u64, + }) + })?; + Ok(rows.collect::>()?) +} + +/// Every annotation, by attachment id (plan P6). +pub(crate) fn query_annotations( + conn: &Connection, +) -> Result, StoreError> { + use ledger_core::{Annotation, AnnotationFields}; + let mut stmt = conn.prepare( + "SELECT attachment_id, receipt_number, payee, amount, date, instrument, category, \ + source, reimbursable_to, note FROM annotations ORDER BY attachment_id", + )?; + let rows = stmt.query_map([], |r| { + let amount: Option = r.get(3)?; + let date: Option = r.get(4)?; + Ok(Annotation { + attachment_id: r.get::<_, i64>(0)? as u64, + fields: AnnotationFields { + receipt_number: r.get::<_, Option>(1)?.map(|v| v as u32), + payee: r.get(2)?, + amount: amount.and_then(|a| a.parse().ok()), + date: date.and_then(|d| NaiveDate::parse_from_str(&d, "%Y-%m-%d").ok()), + instrument: r.get(5)?, + category: r.get(6)?, + source: r.get(7)?, + reimbursable_to: r.get(8)?, + }, + note: r.get(9)?, + }) + })?; + Ok(rows.collect::>()?) +} + /// The base currency of a journal: its first line's, which must be a /// sound `store.created` ([`created_currency`]). pub(crate) fn journal_currency(lines: &[JournalLine]) -> Result { diff --git a/crates/ledger-store/tests/arap_props.proptest-regressions b/crates/ledger-store/tests/arap_props.proptest-regressions index cdae27a..c0c6884 100644 --- a/crates/ledger-store/tests/arap_props.proptest-regressions +++ b/crates/ledger-store/tests/arap_props.proptest-regressions @@ -6,3 +6,4 @@ # everyone who runs the test benefits from these saved cases. cc 704ab49c9bea84359475c3afa56c3354275a353670fc71156fba151f846f4f73 # shrinks to first = [Create(false, 0, 1), Create(false, 0, 1)], then = [Close(1), Hand(false, 1)] cc f678814ec34db3abef40116e17e562fe5abe652c09f12600e19841beaf9d6fe6 # shrinks to first = [Create(false, 0, 13902), Settle { a: 0, b: 0, n: 0, cash: 6676, discount: 551, fee: 0 }], then = [] +cc fade94264cb4e769ef3d72573baff01fc2f78483a32e8424e56251d96c4af57e # shrinks to first = [Create(true, 0, 27747), WriteOff(0, 0, 1), Create(false, 0, 1)], then = [Create(false, 0, 7), WriteOff(3, 66, 2761)] diff --git a/crates/ledger-store/tests/arap_standing.rs b/crates/ledger-store/tests/arap_standing.rs index 1367bfb..461ee53 100644 --- a/crates/ledger-store/tests/arap_standing.rs +++ b/crates/ledger-store/tests/arap_standing.rs @@ -912,6 +912,9 @@ fn issued_pdf_is_filed_with_the_invoice_or_nothing_is() { "entry.posted", "invoice.created", "receipt.attached", + // P6: a filed PDF's text is read at attach time in books that + // declared `attachment_text`, journaled right after the filing. + "attachment.text_extracted", "attachment.linked", ] ); @@ -925,7 +928,7 @@ fn issued_pdf_is_filed_with_the_invoice_or_nothing_is() { assert_eq!(got[5].1["description"], json!("Issued invoice")); assert_eq!(got[5].1["size_bytes"], json!(pdf.len() as u64)); assert_eq!( - got[6].1["subject"], + got[7].1["subject"], json!({"kind": "invoice", "id": inv}), "the copy is linked to the invoice" ); diff --git a/crates/ledger-store/tests/attachments.rs b/crates/ledger-store/tests/attachments.rs index a23afc6..0f62b11 100644 --- a/crates/ledger-store/tests/attachments.rs +++ b/crates/ledger-store/tests/attachments.rs @@ -1452,3 +1452,319 @@ fn a_file_linked_to_an_entry_cannot_also_be_filed_on_it() { s.link_receipt(a, fx.e3).unwrap(); s.verify().unwrap(); } + +// ---------------------------------------------------------------- evidence (P6) + +use ledger_store::format::{ATTACHMENT_TEXT, JournalFormat, current}; +use ledger_store::{BlobStore, ExtractionStatus, JournalWriter}; + +/// Plan P6 step 2: reading a file's text is journaled once per extractor +/// and version; the text is a content-addressed blob; `verify` checks it. +#[test] +fn extraction_is_journaled_once_per_version() { + let dir = test_dir("extract-once"); + let (mut s, fx) = fixture(&dir); + let text = b"Zelle payment to Harbor Design, 12.34, 2026-02-10\n"; + let (id, _, _) = s + .attach_receipt(Some(fx.e2), "zelle.txt", "text/plain", text, None) + .unwrap(); + let x = s.extraction(id).unwrap().expect("an extraction record"); + assert_eq!(x.status, ExtractionStatus::Ok); + assert_eq!(x.pages, 1); + assert_eq!(x.extractor, ledger_import::extract::EXTRACTOR); + assert_eq!(x.version, ledger_import::extract::VERSION); + assert_eq!( + s.extraction_text(id).unwrap().as_deref(), + Some(String::from_utf8_lossy(text).as_ref()) + ); + // One record: journaled once, and the text is a stored blob. + let records = s + .extractions() + .unwrap() + .into_iter() + .filter(|x| x.attachment_id == id) + .count(); + assert_eq!(records, 1); + let sha = x.text_sha256.expect("text"); + assert!(s.stored_blob_shas().unwrap().contains(&sha)); + let seq = s.journal_seq(); + let again = s.extract_attachment(id).unwrap(); + assert_eq!(again.seq, x.seq, "the same record"); + assert_eq!(s.journal_seq(), seq, "a repeat journals nothing"); + s.verify().unwrap(); + // And it survives a reopen: the record and the text are the journal's. + drop(s); + let s = Store::open(&dir).unwrap(); + assert_eq!( + s.extraction_text(id).unwrap().as_deref(), + Some("Zelle payment to Harbor Design, 12.34, 2026-02-10\n") + ); + assert_eq!(s.extractions().unwrap().len(), 1); + s.verify().unwrap(); +} + +/// Plan P6 step 2, R-A: reading a file's text waits for the books to +/// declare `attachment_text`, journaling nothing until then; attaching a +/// file works on old books (the extraction is the backfill's job). +#[test] +fn extract_waits_for_attachment_text() { + let dir = test_dir("extract-upgrade"); + let mut s = Store::create_in_format(&dir, Currency::USD, &JournalFormat::default()).unwrap(); + let (id, _, _) = s + .attach_receipt(None, "note.txt", "text/plain", b"a note", None) + .unwrap(); + assert!(s.extractions().unwrap().is_empty(), "no feature, no record"); + let head = journal_bytes(&dir); + match s.extract_attachment(id) { + Err(StoreError::FormatUpgradePending { need }) => { + assert_eq!(need, format!("the journal feature {ATTACHMENT_TEXT}")) + } + other => panic!("expected FormatUpgradePending, got {other:?}"), + } + assert_eq!(journal_bytes(&dir), head, "a refusal journals nothing"); + s.upgrade_format().unwrap(); + assert_eq!(s.journal_format(), ¤t()); + let x = s.extract_attachment(id).unwrap(); + assert_eq!(x.status, ExtractionStatus::Ok); + assert_eq!(s.extraction_text(id).unwrap().as_deref(), Some("a note")); + // The books replay as written, live and in the projection. + s.verify().unwrap(); + drop(s); + let s = Store::open(&dir).unwrap(); + assert_eq!(s.extraction_text(id).unwrap().as_deref(), Some("a note")); + s.verify().unwrap(); +} + +// Plan P6 step 4: identical bytes are one live attachment, linked to +// every subject they support (property: for any number of filings). +proptest::proptest! { + // Each case makes books on disk (fsync per line): eight is a property + // over the count, not a bulk run. + #![proptest_config(proptest::prelude::ProptestConfig::with_cases(8))] + + #[test] + fn same_bytes_one_record_many_links(n in 1usize..=4) { + let dir = test_dir(&format!("dedup-{n}")); + let (mut s, fx) = fixture(&dir); + let entries = [fx.e1, fx.e2, fx.e3, fx.e4]; + let bytes = b"one receipt, four takers"; + let mut ids = Vec::new(); + for e in entries.iter().take(n) { + let (id, sha, dedup) = s + .attach_receipt(Some(*e), "r.txt", "text/plain", bytes, None) + .unwrap(); + proptest::prop_assert_eq!( + sha, + ledger_store::journal::sha256_hex(bytes) + ); + proptest::prop_assert!(dedup || ids.is_empty()); + ids.push(id); + } + let all: Vec = s.list_attachments(None).unwrap(); + proptest::prop_assert_eq!(all.len(), 1, "one record"); + proptest::prop_assert!(ids.iter().all(|id| *id == all[0].id)); + let links = s.attachment_links(all[0].id).unwrap(); + proptest::prop_assert_eq!(links.len(), n); + proptest::prop_assert_eq!( + s.extractions().unwrap().len(), + 1, + "extraction once" + ); + s.verify().unwrap(); + } +} + +/// Plan P6 step 4: records an older version filed twice open as written; +/// withdrawing one as a duplicate records what it duplicates. +#[test] +fn legacy_duplicates_replay_as_written() { + let dir = test_dir("legacy-dupes"); + let s = Store::create(&dir, Currency::USD).unwrap(); + drop(s); + let bytes = b"filed twice by an older version"; + let (sha, _) = BlobStore::open(&dir.join("blobs")) + .unwrap() + .put(bytes) + .unwrap(); + let mut w = JournalWriter::open(&dir.join(JOURNAL_FILE)).unwrap(); + for (id, name) in [(1, "one.pdf"), (2, "two.pdf")] { + w.append( + "receipt.attached", + json!({ + "attachment_id": id, "blob_sha": sha, "entry_id": null, + "category": "receipts", "file_name": name, + "mime_type": "application/pdf", "size_bytes": bytes.len() as u64, + "description": null, + }), + ) + .unwrap(); + } + drop(w); + let mut s = Store::open(&dir).unwrap(); + let all = s.list_attachments(None).unwrap(); + assert_eq!(all.len(), 2, "both, as written"); + assert_eq!(all[0].blob_sha, all[1].blob_sha); + assert!(all.iter().all(|a| a.duplicate_of.is_none())); + s.withdraw_attachment_duplicate(2, "duplicate of #1", 1) + .unwrap(); + assert_eq!( + s.attachment_by_id(2).unwrap().unwrap().duplicate_of, + Some(1) + ); + s.verify().unwrap(); + drop(s); + let s = Store::open(&dir).unwrap(); + let again = s.attachment_by_id(2).unwrap().unwrap(); + assert_eq!(again.duplicate_of, Some(1)); + assert!(again.withdrawn_at.is_some()); + s.verify().unwrap(); +} + +/// Plan P6 step 5: annotations replay and project, replacing an earlier +/// one, and the engine's next receipt number runs from the highest. +#[test] +fn annotations_replay_and_project() { + let dir = test_dir("annotations"); + let (mut s, fx) = fixture(&dir); + let (id, _, _) = s + .attach_receipt(Some(fx.e2), "r.pdf", "application/pdf", b"%PDF r", None) + .unwrap(); + let first = ledger_core::Annotation { + attachment_id: id, + fields: ledger_core::AnnotationFields { + receipt_number: Some(4), + payee: Some("Harbor Design".into()), + amount: Some(dec("12.34")), + date: Some(date(2026, 2, 1)), + instrument: Some("Check 1234".into()), + category: None, + source: Some("the owner's email".into()), + reimbursable_to: Some("Avery Quill".into()), + }, + note: Some("read on 2026-02-10".into()), + }; + s.annotate_attachment(first.clone()).unwrap(); + assert_eq!(s.ledger().annotation(id), Some(&first), "the live ledger"); + assert_eq!(s.annotations().unwrap(), vec![first.clone()]); + assert_eq!(s.ledger().next_receipt_number(), 5); + // A later annotation replaces it (one projection row, one ledger map + // entry). + let second = ledger_core::Annotation { + attachment_id: id, + fields: ledger_core::AnnotationFields { + receipt_number: Some(6), + payee: Some("Harbor Design LLC".into()), + ..Default::default() + }, + note: None, + }; + s.annotate_attachment(second.clone()).unwrap(); + assert_eq!(s.annotations().unwrap(), vec![second.clone()]); + assert_eq!(s.ledger().next_receipt_number(), 7); + s.verify().unwrap(); + drop(s); + let s = Store::open(&dir).unwrap(); + assert_eq!(s.ledger().annotation(id), Some(&second)); + assert_eq!(s.annotations().unwrap(), vec![second]); + s.verify().unwrap(); +} + +/// Plan P6 step 5: receipt numbers run from the maximum annotated. +#[test] +fn receipt_numbers_run_from_the_max() { + let dir = test_dir("receipt-numbers"); + let (mut s, fx) = fixture(&dir); + let price = |n: u32| ledger_core::Annotation { + attachment_id: 0, + fields: ledger_core::AnnotationFields { + receipt_number: Some(n), + ..Default::default() + }, + note: None, + }; + assert_eq!(s.ledger().next_receipt_number(), 1, "nothing annotated"); + let ids: Vec = (0..3) + .map(|i| { + s.attach_receipt( + Some([fx.e1, fx.e2, fx.e3][i]), + &format!("r{i}.txt"), + "text/plain", + format!("receipt {i}").as_bytes(), + None, + ) + .unwrap() + .0 + }) + .collect(); + for (id, (n, next)) in ids.iter().zip([(3, 4), (9, 10), (7, 10)]) { + let mut a = price(n); + a.attachment_id = *id; + s.annotate_attachment(a).unwrap(); + assert_eq!(s.ledger().next_receipt_number(), next, "{n}"); + } + assert_eq!(s.ledger().next_receipt_number(), 10); + s.verify().unwrap(); +} + +/// Plan P6 step 5: "reimbursed" is derived from `entry.settles` only; +/// nothing about a member-paid entry's shape makes it paid back. +#[test] +fn reimbursed_is_derived_from_settles_only() { + let dir = test_dir("settles"); + let mut s = Store::create(&dir, Currency::USD).unwrap(); + let cash = s + .add_account("1000", "Cash", AccountType::Asset, None, false) + .unwrap(); + let due = s + .add_account( + "2100", + "Due to Members", + AccountType::Liability, + None, + false, + ) + .unwrap(); + let expense = s + .add_account("6000", "Hosting", AccountType::Expense, None, false) + .unwrap(); + let paid = post( + &mut s, + date(2026, 2, 1), + "Hosting, paid by a member", + expense, + due, + "20.00", + ); + let reimb = post( + &mut s, + date(2026, 2, 10), + "Reimburse the member", + due, + cash, + "20.00", + ); + assert!(!s.ledger().settles_entry(paid), "no record yet"); + assert!(s.ledger().entry_settles(reimb).is_empty()); + s.set_entry_settles(reimb, &[paid]).unwrap(); + assert!(s.ledger().settles_entry(paid)); + assert_eq!(s.ledger().entry_settles(reimb), [paid]); + // Refusals journal nothing and change no record. + let head = journal_bytes(&dir); + for (entry, settles) in [ + (paid, vec![paid]), + (reimb, vec![paid, paid]), + (reimb, vec![999]), + ] { + assert!( + s.set_entry_settles(entry, &settles).is_err(), + "{entry} {settles:?}" + ); + } + assert_eq!(journal_bytes(&dir), head, "refusals journal nothing"); + s.verify().unwrap(); + drop(s); + let s = Store::open(&dir).unwrap(); + assert!(s.ledger().settles_entry(paid)); + assert_eq!(s.ledger().entry_settles(reimb), [paid]); + s.verify().unwrap(); +} diff --git a/crates/ledger-store/tests/fixtures/old-books/p6_evidence/blobs/16/97/1697fa59fa7f1f7e6d4e3b53940e9208596242b8451ddb01bef8b00b71040c2b b/crates/ledger-store/tests/fixtures/old-books/p6_evidence/blobs/16/97/1697fa59fa7f1f7e6d4e3b53940e9208596242b8451ddb01bef8b00b71040c2b new file mode 100644 index 0000000..e0f0540 --- /dev/null +++ b/crates/ledger-store/tests/fixtures/old-books/p6_evidence/blobs/16/97/1697fa59fa7f1f7e6d4e3b53940e9208596242b8451ddb01bef8b00b71040c2b @@ -0,0 +1,3 @@ +# Records + +Kept with the company's papers. diff --git a/crates/ledger-store/tests/fixtures/old-books/p6_evidence/blobs/2d/66/2d6666a14d257ebe2f29c5a10380afaafed93efee2b80d79af3d3d53c10af699 b/crates/ledger-store/tests/fixtures/old-books/p6_evidence/blobs/2d/66/2d6666a14d257ebe2f29c5a10380afaafed93efee2b80d79af3d3d53c10af699 new file mode 100644 index 0000000..db83f71 --- /dev/null +++ b/crates/ledger-store/tests/fixtures/old-books/p6_evidence/blobs/2d/66/2d6666a14d257ebe2f29c5a10380afaafed93efee2b80d79af3d3d53c10af699 @@ -0,0 +1 @@ +Zelle payment to A. Quill, 20.00, 2026-02-10 diff --git a/crates/ledger-store/tests/fixtures/old-books/p6_evidence/blobs/58/37/5837c77f55b72825a8a6be69e2aecc3e962fba90277e12547ae2d6b7f1ad0435 b/crates/ledger-store/tests/fixtures/old-books/p6_evidence/blobs/58/37/5837c77f55b72825a8a6be69e2aecc3e962fba90277e12547ae2d6b7f1ad0435 new file mode 100644 index 0000000..09f2fdb Binary files /dev/null and b/crates/ledger-store/tests/fixtures/old-books/p6_evidence/blobs/58/37/5837c77f55b72825a8a6be69e2aecc3e962fba90277e12547ae2d6b7f1ad0435 differ diff --git a/crates/ledger-store/tests/fixtures/old-books/p6_evidence/expected.json b/crates/ledger-store/tests/fixtures/old-books/p6_evidence/expected.json new file mode 100644 index 0000000..916e741 --- /dev/null +++ b/crates/ledger-store/tests/fixtures/old-books/p6_evidence/expected.json @@ -0,0 +1,187 @@ +{ + "assets": [], + "bills": [], + "entries": [ + { + "date": "2026-01-01", + "description": "Opening capital", + "id": 1, + "lines": [ + { + "account": 1, + "amount": { + "amount": "500.00", + "currency": [ + 85, + 83, + 68 + ] + }, + "side": "Debit" + }, + { + "account": 3, + "amount": { + "amount": "500.00", + "currency": [ + 85, + 83, + 68 + ] + }, + "side": "Credit" + } + ], + "reversal_of": null, + "status": "Posted" + }, + { + "date": "2026-02-01", + "description": "Web hosting, paid by a member", + "id": 2, + "lines": [ + { + "account": 4, + "amount": { + "amount": "20.00", + "currency": [ + 85, + 83, + 68 + ] + }, + "side": "Debit" + }, + { + "account": 2, + "amount": { + "amount": "20.00", + "currency": [ + 85, + 83, + 68 + ] + }, + "side": "Credit" + } + ], + "reversal_of": null, + "status": "Posted" + }, + { + "date": "2026-02-10", + "description": "Reimburse the member", + "id": 3, + "lines": [ + { + "account": 2, + "amount": { + "amount": "20.00", + "currency": [ + 85, + 83, + 68 + ] + }, + "side": "Debit" + }, + { + "account": 1, + "amount": { + "amount": "20.00", + "currency": [ + 85, + 83, + 68 + ] + }, + "side": "Credit" + } + ], + "reversal_of": null, + "status": "Posted" + }, + { + "date": "2026-03-01", + "description": "March rent", + "id": 4, + "lines": [ + { + "account": 4, + "amount": { + "amount": "100.00", + "currency": [ + 85, + 83, + 68 + ] + }, + "side": "Debit" + }, + { + "account": 1, + "amount": { + "amount": "100.00", + "currency": [ + 85, + 83, + 68 + ] + }, + "side": "Credit" + } + ], + "reversal_of": null, + "status": "Posted" + } + ], + "entry_parties": [], + "invoices": [], + "payroll_runs": [], + "periods": [], + "recurring": [], + "schedules": [], + "snapshot": { + "Ok": { + "balance_sheet": { + "assets": "380", + "balanced": true, + "current_earnings": "-120", + "equity": "380", + "liabilities": "0" + }, + "profit_and_loss": { + "expenses": "120", + "net_income": "-120", + "revenue": "0" + }, + "trial_balance": { + "lines": [ + { + "code": "1000", + "credit": "0", + "debit": "380", + "name": "Cash" + }, + { + "code": "3000", + "credit": "500", + "debit": "0", + "name": "Capital" + }, + { + "code": "6000", + "credit": "0", + "debit": "120", + "name": "Rent" + } + ], + "total_credits": "500", + "total_debits": "500" + } + } + }, + "verify": { + "Ok": null + } +} diff --git a/crates/ledger-store/tests/fixtures/old-books/p6_evidence/journal.jsonl b/crates/ledger-store/tests/fixtures/old-books/p6_evidence/journal.jsonl new file mode 100644 index 0000000..8437ef1 --- /dev/null +++ b/crates/ledger-store/tests/fixtures/old-books/p6_evidence/journal.jsonl @@ -0,0 +1,25 @@ +{"payload":{"base_currency":"USD"},"prev":"0000000000000000000000000000000000000000000000000000000000000000","seq":1,"ts":"2026-10-27T15:18:09Z","type":"store.created"} +{"payload":{"features":["account_renamed","asset_basis","depreciation_catch_up","disposal_before_service","document_line_projects","document_standing","documents_v2","members_cumulative","month_reposting","party_creditors","payment_instruments","projects","require_tax_category","schedule_convention","tax_categories_gain_exempt","vendor_1099_decisions"],"format":2},"prev":"7f124ed3e91db7042661abf4e5007b1dcae9903a767bea76e7e727a96075bfa6","seq":2,"ts":"2026-10-27T15:18:09Z","type":"journal.format_set"} +{"payload":{"account_type":"Asset","code":"1000","currency":"USD","id":1,"name":"Cash","parent":null,"placeholder":false},"prev":"955f321d1a0bbd460640ba5339b2d3e1313038f3ba615e5ffff223ba3c8c728f","seq":3,"ts":"2026-10-27T15:18:09Z","type":"account.added"} +{"payload":{"account_type":"Liability","code":"2100","currency":"USD","id":2,"name":"Due to Members","parent":null,"placeholder":false},"prev":"7b619dff5f0fb25b516b96b459b39ef67713bef25689946837a28f58e7692db5","seq":4,"ts":"2026-10-27T15:18:09Z","type":"account.added"} +{"payload":{"account_type":"Equity","code":"3000","currency":"USD","id":3,"name":"Capital","parent":null,"placeholder":false},"prev":"d55d7fb14bbfbd1aa43f067617901d7086ecbfa16a693f0aa651b98ba525e4f8","seq":5,"ts":"2026-10-27T15:18:09Z","type":"account.added"} +{"payload":{"account_type":"Expense","code":"6000","currency":"USD","id":4,"name":"Rent","parent":null,"placeholder":false},"prev":"6cd464539b1b1aa60ae1093728da6a48658cc7bf3f962c88a21c863b85b17523","seq":6,"ts":"2026-10-27T15:18:09Z","type":"account.added"} +{"payload":{"date":"2026-01-01","description":"Opening capital","id":1},"prev":"ed316ff3ac6c630502dd56101b39b53fac83e4554fd78e271c91d0d468ae6c93","seq":7,"ts":"2026-10-27T15:18:09Z","type":"entry.created"} +{"payload":{"account":1,"amount":"500.00","currency":"USD","id":1,"side":"debit"},"prev":"cf19ca8e1f8992a73c249c3d5e06997b681799a19d39261082d66ecee54a7c78","seq":8,"ts":"2026-10-27T15:18:09Z","type":"entry.line_added"} +{"payload":{"account":3,"amount":"500.00","currency":"USD","id":1,"side":"credit"},"prev":"31d98c34346a4ca836f1b7a51e993a7c8b682eaab36f061d92e8659eafcf3691","seq":9,"ts":"2026-10-27T15:18:09Z","type":"entry.line_added"} +{"payload":{"date":"2026-01-01","description":"Opening capital","id":1,"lines":[{"account":1,"amount":"500.00","currency":"USD","side":"debit"},{"account":3,"amount":"500.00","currency":"USD","side":"credit"}],"reversal_of":null},"prev":"1589ea4608834aab2019ebd960438eed2ebd843abef29fbcf2174f65b88a34e8","seq":10,"ts":"2026-10-27T15:18:09Z","type":"entry.posted"} +{"payload":{"date":"2026-02-01","description":"Web hosting, paid by a member","id":2},"prev":"ea67596a958a0c100bc81e4d4b7058c3145803ea765a3989d3da58eef343c775","seq":11,"ts":"2026-10-27T15:18:09Z","type":"entry.created"} +{"payload":{"account":4,"amount":"20.00","currency":"USD","id":2,"side":"debit"},"prev":"1e78ebd8c378a723511d3e054f216c02587d9c0795a2958b64a7ef681405301f","seq":12,"ts":"2026-10-27T15:18:09Z","type":"entry.line_added"} +{"payload":{"account":2,"amount":"20.00","currency":"USD","id":2,"side":"credit"},"prev":"06d014884d44765a38cb4ca9220cef0b913ef8a8b561997fdea7d59c0094dc0e","seq":13,"ts":"2026-10-27T15:18:09Z","type":"entry.line_added"} +{"payload":{"date":"2026-02-01","description":"Web hosting, paid by a member","id":2,"lines":[{"account":4,"amount":"20.00","currency":"USD","side":"debit"},{"account":2,"amount":"20.00","currency":"USD","side":"credit"}],"reversal_of":null},"prev":"1863ad9b2fb040fd24c1398b7ec995f083153fbacc5e252e88bcd98541e3c33a","seq":14,"ts":"2026-10-27T15:18:10Z","type":"entry.posted"} +{"payload":{"date":"2026-02-10","description":"Reimburse the member","id":3},"prev":"6e7ec8d7738b85d7e79548f22eb6e314a370a31757c1651a67fa752b7adc4651","seq":15,"ts":"2026-10-27T15:18:10Z","type":"entry.created"} +{"payload":{"account":2,"amount":"20.00","currency":"USD","id":3,"side":"debit"},"prev":"a26929387cbba06f0e89c9b1dd4118320f22be51d8ee499ee398e411f16b890f","seq":16,"ts":"2026-10-27T15:18:10Z","type":"entry.line_added"} +{"payload":{"account":1,"amount":"20.00","currency":"USD","id":3,"side":"credit"},"prev":"b2e9405890eb89ae0e5afd17d3e5cf741e330960f3f01449bd7a82c2e1dea440","seq":17,"ts":"2026-10-27T15:18:10Z","type":"entry.line_added"} +{"payload":{"date":"2026-02-10","description":"Reimburse the member","id":3,"lines":[{"account":2,"amount":"20.00","currency":"USD","side":"debit"},{"account":1,"amount":"20.00","currency":"USD","side":"credit"}],"reversal_of":null},"prev":"94f2dbba114035ebe433c985c6b4f5ace26bd4588903fff15bef8717ab82bc77","seq":18,"ts":"2026-10-27T15:18:10Z","type":"entry.posted"} +{"payload":{"date":"2026-03-01","description":"March rent","id":4},"prev":"b1c37b5614c19fdbe10e16ded9072b31bc0f2576cd6419a86c14d258871ba6dc","seq":19,"ts":"2026-10-27T15:18:10Z","type":"entry.created"} +{"payload":{"account":4,"amount":"100.00","currency":"USD","id":4,"side":"debit"},"prev":"66f50f061925b1be9ad46a015ab7359729324b58419ee97abdc2e430c19cd335","seq":20,"ts":"2026-10-27T15:18:10Z","type":"entry.line_added"} +{"payload":{"account":1,"amount":"100.00","currency":"USD","id":4,"side":"credit"},"prev":"1e5e7bd1e14131c980d43b5da0fb6a21673dd3f7120dfe87873cd7e88eb49340","seq":21,"ts":"2026-10-27T15:18:10Z","type":"entry.line_added"} +{"payload":{"date":"2026-03-01","description":"March rent","id":4,"lines":[{"account":4,"amount":"100.00","currency":"USD","side":"debit"},{"account":1,"amount":"100.00","currency":"USD","side":"credit"}],"reversal_of":null},"prev":"c3ee5fc9f8cf08b452969df15921f6bcaf61514c608971b6180e8b89bdbabfbb","seq":22,"ts":"2026-10-27T15:18:10Z","type":"entry.posted"} +{"payload":{"attachment_id":1,"blob_sha":"5837c77f55b72825a8a6be69e2aecc3e962fba90277e12547ae2d6b7f1ad0435","category":"receipts","description":"March rent invoice","entry_id":4,"file_name":"receipt.pdf","mime_type":"application/pdf","size_bytes":20748},"prev":"4820680a8b8f00fbfee12dcdb515f6c7232fe002b17f6a0d875cbc7b37f9301c","seq":23,"ts":"2026-10-27T15:18:10Z","type":"receipt.attached"} +{"payload":{"attachment_id":2,"blob_sha":"2d6666a14d257ebe2f29c5a10380afaafed93efee2b80d79af3d3d53c10af699","category":"receipts","description":null,"entry_id":3,"file_name":"zelle.txt","mime_type":"text/plain","size_bytes":45},"prev":"06043343802dce570e36d0502cc359bc05dd00f5ccf0831f34c516648ee0c9dc","seq":24,"ts":"2026-10-27T15:18:10Z","type":"receipt.attached"} +{"payload":{"attachment_id":3,"blob_sha":"1697fa59fa7f1f7e6d4e3b53940e9208596242b8451ddb01bef8b00b71040c2b","category":"other","description":null,"entry_id":null,"file_name":"note.md","mime_type":"text/markdown","size_bytes":43},"prev":"9cad59cd4384dc0769bc925032cad7366c814e827d094795404b9b6d9e5ce861","seq":25,"ts":"2026-10-27T15:18:10Z","type":"document.filed"} diff --git a/crates/ledger-store/tests/spec_store.rs b/crates/ledger-store/tests/spec_store.rs index 2c0fe05..447724b 100644 --- a/crates/ledger-store/tests/spec_store.rs +++ b/crates/ledger-store/tests/spec_store.rs @@ -940,11 +940,14 @@ fn attach_stores_under_sha256() { assert_eq!(b.store.verify_blobs().unwrap(), 1); } -/// Oracle: Store::attach_receipt INVARIANTS 2 and 3 — the same bytes filed -/// twice share one stored file, the second says deduplicated, and each -/// filing is its own journaled attachment with its name, type, description. +/// Oracle: Store::attach_receipt INVARIANTS 2 and 3, as plan P6 step 4 +/// changes them — the same bytes filed twice share one stored file and one +/// live attachment, the second says deduplicated, and that attachment is +/// linked to the second subject (its name, type and description are the +/// first filing's). Records an older version filed twice stay as written +/// (`legacy_duplicates_replay_as_written`). #[test] -fn attach_dedups_bytes_but_not_attachments() { +fn same_bytes_are_one_attachment_linked_to_both() { let dir = test_dir("attach-dedup"); let mut b = books(&dir); let bytes = b"same bytes"; @@ -962,52 +965,61 @@ fn attach_dedups_bytes_but_not_attachments() { .store .attach_receipt(Some(b.e2), "two.png", "image/png", bytes, Some("second")) .unwrap(); + assert_eq!((a1, a2), (a1, a1), "one attachment"); assert_eq!(s1, s2); assert!(!d1); assert!(d2); - assert_ne!(a1, a2); let blob_files: Vec<_> = tree(&dir) .into_keys() .filter(|p| p.starts_with("blobs")) .collect(); assert_eq!(blob_files.len(), 1, "one stored file: {blob_files:?}"); let all = b.store.list_attachments(None).unwrap(); - let one = all.iter().find(|a| a.id == a1).expect("first listed"); - let two = all.iter().find(|a| a.id == a2).expect("second listed"); + assert_eq!(all.len(), 1, "one record: {all:?}"); + let one = all.iter().find(|a| a.id == a1).expect("listed"); assert_eq!(one.entry_id, Some(b.e1)); - assert_eq!(two.entry_id, Some(b.e2)); assert_eq!( (one.file_name.as_str(), one.mime_type.as_str()), ("one.pdf", "application/pdf") ); assert_eq!(one.description.as_deref(), Some("first")); - assert_eq!(two.description.as_deref(), Some("second")); assert_eq!(one.size_bytes, bytes.len() as u64); assert_eq!(one.blob_sha, s1); + // The second subject is linked, and counts as its receipt. + assert!( + b.store + .attachment_links(a1) + .unwrap() + .contains(&ledger_store::Subject::Entry(b.e2)) + ); + assert_eq!( + b.store + .attachments_for(ledger_store::Subject::Entry(b.e2)) + .unwrap() + .len(), + 1 + ); + // One filing line: the second added a link, not a record. let attached: Vec = parsed_lines(&dir) .into_iter() .filter(|l| l["type"] == "receipt.attached") .collect(); - assert_eq!(attached.len(), 2); - for (l, (name, mime, desc)) in attached.iter().zip([ - ("one.pdf", "application/pdf", "first"), - ("two.png", "image/png", "second"), - ]) { - for needle in [name, mime, desc, s1.as_str()] { - assert!( - contains_value(&l["payload"], needle), - "{needle} missing from {l}" - ); - } + assert_eq!(attached.len(), 1); + for needle in ["one.pdf", "application/pdf", "first", s1.as_str()] { + assert!( + contains_value(&attached[0]["payload"], needle), + "{needle} missing from {}", + attached[0] + ); } - assert_eq!(b.store.attachment_bytes(a2).unwrap(), bytes); + assert_eq!(b.store.attachment_bytes(a1).unwrap(), bytes); } -/// Oracle: Store::attach_receipt INVARIANT 3 — each filing is its own -/// attachment with ITS name and type, even when its bytes deduplicate -/// against an earlier filing. +/// Oracle: Store::attach_receipt INVARIANT 3, as plan P6 step 4 changes +/// it — one file, one record: the first filing's own name and type stay, +/// in the writer and in the viewer. #[test] -fn attach_dedup_keeps_each_filings_name_and_type() { +fn the_first_filings_name_and_type_stay() { let dir = test_dir("attach-dedup-name"); let mut b = books(&dir); let bytes = b"same bytes, two names"; @@ -1019,28 +1031,25 @@ fn attach_dedup_keeps_each_filings_name_and_type() { .store .attach_receipt(Some(b.e2), "two.png", "image/png", bytes, None) .unwrap(); + assert_eq!((a1, a2), (a1, a1)); assert!(d2); let all = b.store.list_attachments(None).unwrap(); + assert_eq!(all.len(), 1); let one = all.iter().find(|a| a.id == a1).unwrap(); - let two = all.iter().find(|a| a.id == a2).unwrap(); assert_eq!( (one.file_name.as_str(), one.mime_type.as_str()), ("one.pdf", "application/pdf") ); - assert_eq!( - (two.file_name.as_str(), two.mime_type.as_str()), - ("two.png", "image/png") - ); let snap = Snapshot::load(&dir).unwrap(); - let s2 = snap + let s1 = snap .attachments(None) .unwrap() .into_iter() - .find(|a| a.id == a2) + .find(|a| a.id == a1) .unwrap(); assert_eq!( - (s2.file_name.as_str(), s2.mime_type.as_str()), - ("two.png", "image/png") + (s1.file_name.as_str(), s1.mime_type.as_str()), + ("one.pdf", "application/pdf") ); } diff --git a/crates/ledger-store/tests/store.rs b/crates/ledger-store/tests/store.rs index bd01577..d89f8ae 100644 --- a/crates/ledger-store/tests/store.rs +++ b/crates/ledger-store/tests/store.rs @@ -1893,6 +1893,21 @@ fn books_written_under_older_rules_still_open() { json!({"id": 1, "name": "Tobin Harlow Supply"}), ) .unwrap(); + // P6's new shapes, hand-written as a later version writes them: a file's + // extraction record, an annotation of it, and a settlement. This reader + // takes them; the version before it stopped at the declaration. + w.append( + "attachment.text_extracted", + json!({"attachment_id": 5, "text_sha256": null, "extractor": "wanderbooks", + "version": "1", "pages": 0, "status": "none"}), + ) + .unwrap(); + w.append( + "attachment.annotated", + json!({"attachment_id": 5, "fields": {"receipt_number": 3, "payee": "Acme"}, + "note": "read by hand"}), + ) + .unwrap(); w.append( "entry.created", json!({"id": 2, "date": "2026-04-01", "description": "Invoice 1"}), @@ -2070,6 +2085,30 @@ fn books_written_under_older_rules_still_open() { "amount": "30.00", "reason": "uncollectible"}), ) .unwrap(); + // P6's new shapes, hand-written as a later version writes them: an + // entry a member was reimbursed by, and the record of what it settles. + w.append( + "entry.created", + json!({"id": 7, "date": "2026-03-02", "description": "Reimburse"}), + ) + .unwrap(); + for side in ["debit", "credit"] { + w.append( + "entry.line_added", + json!({"id": 7, "account": 2, "side": side, "amount": "5.00", "currency": "USD"}), + ) + .unwrap(); + } + w.append( + "entry.posted", + json!({"id": 7, "date": "2026-03-02", "description": "Reimburse", + "lines": [{"account": 2, "side": "debit", "amount": "5.00", "currency": "USD"}, + {"account": 2, "side": "credit", "amount": "5.00", "currency": "USD"}], + "reversal_of": null}), + ) + .unwrap(); + w.append("entry.settles", json!({"entry_id": 7, "settles": [1]})) + .unwrap(); drop(w); // Opening rebuilds from the journal; it must accept all of it. let _ = std::fs::remove_file(dir.join(ledger_store::db_file())); @@ -2086,6 +2125,21 @@ fn books_written_under_older_rules_still_open() { "108.34567".parse::().unwrap() ); // The new shapes replay as written: the party's facts and the setting. + // P6's shapes replayed: the extraction, the annotation and the + // settlement. + let extractions = store.extractions().unwrap(); + assert_eq!(extractions.len(), 1); + assert_eq!(extractions[0].attachment_id, 5); + assert_eq!(extractions[0].status, ledger_store::ExtractionStatus::None); + let annotation = store + .ledger() + .annotation(5) + .expect("the annotation, replayed"); + assert_eq!(annotation.fields.receipt_number, Some(3)); + assert_eq!(annotation.fields.payee.as_deref(), Some("Acme")); + assert_eq!(store.ledger().next_receipt_number(), 4); + assert_eq!(store.ledger().entry_settles(7), [1]); + assert!(store.ledger().settles_entry(1)); let acme = store.ledger().get_party(1).expect("the party"); assert_eq!(acme.email.as_deref(), Some("billing@acme.example")); assert_eq!(acme.phone.as_deref(), Some("555-0142")); diff --git a/docs/architecture.md b/docs/architecture.md index 16f855a..4172433 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -115,6 +115,24 @@ the agent's cannot drift apart. The crate depends inward (`ledger-core`, `ledger-store`, `ledger-config`, `ledger-import`, `ledger-vault`), never on `ledger-mcp` or `ledger-gui`. +## The evidence pipeline (`ledger-import`, `ledger-core`, `ledger-store`, P6) + +One extractor lives in `ledger-import` (`src/extract.rs`, D-53: in-process, +no new crate): a PDF's text and pages, an XLSX's cells as exact decimal +strings, and plain text; the caps are 25 MiB in, 30 pages and 2 MiB of +text, and a reader panic is caught there. `ledger-store` depends on +`ledger-import` so the writer can read a file as it is filed and journal +`attachment.text_extracted` (content-addressed text blob, extractor, +version, pages, `ok`/`none`/`failed`), one record per attachment, extractor +and version; `Store::verify` checks the text blob's hash and never +re-derives it. The server's filing scan (`tin_files`) and the viewer's PDF +rasterizer read through the same module, so a PDF's text is read one way. +Annotations and settlements are ledger state (`ledger_core::evidence`, +`Ledger::differs_from`), projected into schema 16's `annotations` table; +extractions have a table of their own. The viewer's Documents page groups +by category and filters by subject; Needs attention reads both through +`PackageBooks`. + ## The journal format gate (`ledger-store`, P0) Every journal line passes a gate after its parse, seq and chain checks @@ -187,6 +205,8 @@ groups are: - `write_offs` — `books.write_off_invoice`, the bad debt (P4b S7; rules.md B-41), kept apart from `documents` so S4.4's document tools and this one land in different files; +- `evidence` — finding a file by its hash, number or text, annotating it, + recording what a payment settles, and consolidating duplicates (P6); - `settings` — the handbook, `books.settings` and the recorded FX rates (the module is `settings_tools`, since `crate::settings` is the live settings); diff --git a/docs/mcp-interface.md b/docs/mcp-interface.md index c16f164..4bae3d7 100644 --- a/docs/mcp-interface.md +++ b/docs/mcp-interface.md @@ -365,7 +365,8 @@ that changes anything also has a `destructiveHint`, true for these 30: `books.set_tin` (both overwrite a vault record), `books.purge_tin`, `books.create_invoice`, `books.create_bill`, `books.receive_payment`, `books.pay_bill`, `books.void_document`, `books.write_off_invoice`, -`books.run_payroll`, `books.remit_payroll_taxes`; and an +`books.run_payroll`, `books.remit_payroll_taxes`, +`books.consolidate_duplicates`; and an `idempotentHint` on the ten that a repeat with the same arguments leaves the journal and the vault as they were: `books.verify`, `books.run_depreciation`, `books.run_schedules`, `books.run_recurring`, @@ -1598,22 +1599,82 @@ and no control or invisible (Unicode format) characters; a MIME type subject it is linked to, its filing entry first). Withdrawn attachments are omitted unless `include_withdrawn` is true. `entry_id` lists the files filed on the entry and linked to it. -- `books.get_receipt { attachment_id, view?: "meta" | "bytes" }` → `{ - attachment_id, sha256, entry_id?, category, file_name, mime_type, - size_bytes, description?, links, file? }`. `meta` (the default) returns - no file. `bytes` adds `file`, the file handed over (a `HandedFile`, read - hash-verified, whatever its size: D-46), whose `download` saves a copy - as a new file in a folder the human names. The stored file is also - fetched by `wanderbooks download --book B --to ` (`GET +- `books.get_receipt { attachment_id, view?: "meta" | "text" | + "page_image" | "bytes", page?, max_px? }` → `{ attachment_id, sha256, + entry_id?, category, file_name, mime_type, size_bytes, description?, + links, text?, text_truncated, text_status, pages, receipt_number?, + annotation?, file? }`. `meta` (the default) returns no file: the first + 8 KiB of the file's extracted text (`text`, `text_truncated` when cut), + its page count (`pages`), and `text_status` (`ok`, `none`, `failed`, or + `not_extracted` for books filed before this version: `wanderbooks books + extract`). `text` gives the whole extracted text (past 64 KiB it comes + as a handed-over `.txt` instead). `page_image` with `page` (1 is the + first) and `max_px` (default 1600, longest side) gives one page as a + PNG, handed over (D-46; D-15: no OCR, so a picture-only receipt is read + from its pages). Read the text first; a page picture only when the text + is empty or looks wrong; never `bytes` into context. `bytes` adds + `file`, the file handed over (a `HandedFile`, read hash-verified, + whatever its size: D-46), whose `download` saves a copy as a new file + in a folder the human names. The stored file is also fetched by + `wanderbooks download --book B --to ` (`GET /b/{book}/a/{attachment}`, books:read, "Transports"). - `books.file_document { category: "formation" | "legal" | "tax" | "accounting" | "statement" | "contract" | "photo" | "warranty" | - "import_source" | "other", file_name?, mime_type?, upload_id, - description?, link_to? }` → `{ attachment_id, - sha256, deduplicated, links }`. The bytes, name and type as in - `books.attach_receipt`. The Drive-folder replacement (today's 01 - Formation / 02 Legal / 03 Tax / 04 Accounting). Never a W-9: it shows - the full TIN (`books.set_w9 w9_upload_id`). + "import_source" | "notices" | "correspondence" | "plans" | "history" | + "other", file_name?, mime_type?, upload_id, description?, link_to? }` → + `{ attachment_id, sha256, deduplicated, links }`. The bytes, name and + type as in `books.attach_receipt`. Every company record has a home + (D-20): the Drive folders' replacement, and formation papers, + registrations, plans, notices, correspondence and history besides what + the CPA asks for. Never a W-9: it shows the full TIN (`books.set_w9 + w9_upload_id`). +- **One file, one record** (plan P6 step 4): filing bytes already stored + (and not withdrawn) returns that attachment's `attachment_id` with + `deduplicated: true` and links it to the new subject, instead of filing + a second record. Records an older version filed twice stay as written + until `books.consolidate_duplicates`. + +## Tools — evidence + +What the books know about a filed file beyond its bytes (plan P6): its +extracted text, the facts a human read off it, and which entries a payment +settles. Extraction is journaled as it is made (`attachment.text_extracted`, +content-addressed, one record per attachment, extractor and version): an +extractor upgrade adds a record, never replaces one, and `books.verify` +checks the text blob's presence and hash. + +- `books.find_attachment { sha256 | receipt_number | text }` → `{ + attachments: [...] }`. Exactly one of the three: the file's exact + SHA-256 (64 hex digits), the number recorded with + `books.annotate_attachment`, or words its extracted text says + (case-insensitively). Withdrawn files are listed too, marked, so a + duplicate is found before it is consolidated. Each attachment is as in + `books.list_receipts`, with its links. +- `books.annotate_attachment { attachment_id, fields { receipt_number?, + payee?, amount?, date?, instrument?, category?, source?, + reimbursable_to? }, note? }` → `{ attachment_id, fields, + next_receipt_number }`. The closed list of what a human reads off a + file: `amount` an exact decimal string, `date` YYYY-MM-DD, `instrument` + a payment method's id or label. It changes no amount, no file and no + posting; a later annotation replaces it, journaled as + `attachment.annotated`. `next_receipt_number` is one past the highest + number annotated, so numbers run without reuse. At least one field or + the note is required; an unknown or withdrawn attachment is refused. +- `books.set_entry_settles { entry_id, settles }` → `{ ok, entry_id, + settles }`. Records which entries an entry pays back (a reimbursement + settling the expenses a member paid personally), journaled as + `entry.settles`; an empty list clears it. Every named entry must exist, + none may be the paying entry itself or be named twice. It changes no + amount. "The member was paid back" is read from this record only, never + guessed (IZ-18); Needs attention asks until it is recorded. +- `books.consolidate_duplicates { confirmed? }` → `{ status: "preview" | + "consolidated", groups: [{ sha256, keep, withdraw, links }], withdrawn + }`. For every set of attachments holding the same bytes it keeps the + oldest (`keep`), links it to every subject the others had, unlinks the + others and withdraws them as duplicates (`attachment.withdrawn` with + `duplicate_of`; nothing is deleted). A posted entry that would be left + with no receipt the policy requires refuses the whole call, journaling + nothing. Without `confirmed`, a preview that changes nothing. ## Receipt enforcement @@ -1868,8 +1929,12 @@ Hand-rolled on `ServerHandler` (rmcp 3.x has no resource macros): this month so far - `books://handbook` (the index) and `books://handbook/{section}` (one per section, Markdown) -- `books://receipts/{id}` — receipt metadata (as `books.get_receipt` +- `books://receipts/{id}` (an alias of `books://attachments/{id}`) — + receipt metadata and the first 8 KiB of its text (as `books.get_receipt` view `meta`; no bytes) +- `books://attachments/{id}/page/{n}` — page `n` (from 1) as a PNG blob, + the first binary resource (plan P6; D-46: the bytes are the resource's + contents, never a path) Resources exist so read-only clients (dashboards, the viewer, auditors) never need tool credentials. Tested end to end through a real diff --git a/docs/parity.md b/docs/parity.md index 79d5443..d8c257f 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -173,3 +173,20 @@ stay derived, and its evidence of worthlessness filed in the same call -3(a)). A same-year recovery is the reversal D-49 records, not a new movement; bills are never written off. No row above changed status (the A/R and A/P rows already say `done`), so the floor stands (10 done). + +## P6: the evidence pipeline (2026-10-26–27) + +No semantic feature of the catalog above moved, so no status changed and +the floor stands (10 done): P6 changes how evidence is kept, not how a +figure is computed. It adds one in-process extractor (a PDF's text and +pages, an XLSX's cells as exact decimal strings, plain text; 25 MiB, 30 +pages, 2 MiB of text), journals each extraction as its own record (one per +attachment, extractor and version, its text a content-addressed blob whose +hash `books.verify` checks), holds identical bytes as one live attachment +(linked to every subject, legacy duplicates consolidated with +`duplicate_of`), records what a human read off a file (the closed +annotation field list, engine receipt numbers running from the max) and +what a payment settles (`entry.settles`, the only source of "reimbursed"), +and gives company records four more homes (`notices`, `correspondence`, +`plans`, `history`, D-20). Rules.md B-44 states it; `docs/testing.md`'s P6 +section lists its tests and the P12 mutation targets. diff --git a/docs/receipt-store.md b/docs/receipt-store.md index bb4ba54..07e9cfa 100644 --- a/docs/receipt-store.md +++ b/docs/receipt-store.md @@ -57,13 +57,11 @@ apply is safe — on startup the applier replays every journal line with - `seq`: 1-based, gapless, assigned by the writer. - `ts`: UTC, RFC 3339. Recorded, never trusted for ordering (`seq` orders). - `type`: `account.added`, `entry.posted`, `entry.reversed`, `invoice.created`, - `bill.created`, `receipt.attached`, `receipt.linked`, `receipt.unlinked`, - `document.filed`, `attachment.withdrawn`, `invoice.sent` (a recorded - send: a document's record, never a ledger line or a figure; P4b S5) - (plus the A/R, A/P, asset, period, payroll, and class events listed on - `JournalLine`). - `invoice.written_off`, `bill.created`, `receipt.attached`, `receipt.linked`, `receipt.unlinked`, - `document.filed`, `attachment.withdrawn` (plus the A/R, A/P, asset, + `bill.created`, `invoice.sent` (a recorded send: a document's record, + never a ledger line or a figure; P4b S5), `invoice.written_off` (P4b S7), + `receipt.attached`, `receipt.linked`, `receipt.unlinked`, + `document.filed`, `attachment.withdrawn`, `attachment.text_extracted`, + `attachment.annotated`, `entry.settles` (P6) (plus the A/R, A/P, asset, period, payroll, and class events listed on `JournalLine`). - `payload`: the full domain facts (account fields, entry with lines, blob sha, ...). Everything the applier needs; nothing it must look up elsewhere. @@ -114,9 +112,12 @@ a backup is then proven where it lands, by a drill (docs/mcp-interface.md, - **Immutable.** Blobs are never modified or deleted by the application. A retention/deletion policy is an explicit future human decision, not an accident of implementation. -- **Deduplicated.** Attaching the same receipt twice stores one blob and two - attachment rows. `attach_receipt` / `file_document` return - `deduplicated: true` when the blob already existed. +- **Deduplicated.** The same bytes are one blob, and since P6 one live + attachment record: filing them again returns that record (with + `deduplicated: true`) and links it to the new subject instead of adding + a second row. Records an older version filed twice stay as written until + `books.consolidate_duplicates` links the oldest to every subject the + others had and withdraws the rest (`duplicate_of`). - **Limits.** 25 MiB per file, as an upload (`books.begin_upload`): the only way a file comes in (D-46: nothing is given inline). The name and MIME type are the client's, checked when the file is filed: a name of 1 to 255 bytes, no `/` or `\`, not @@ -135,17 +136,33 @@ a backup is then proven where it lands, by a drill (docs/mcp-interface.md, exists, then linked with `books.link_receipt`. - `category` mirrors today's Drive layout: `formation`, `legal`, `tax`, `accounting` (today's 01 Formation / 02 Legal / 03 Tax / 04 Accounting), - plus `receipts`. + plus `receipts`, and every other company record (D-20): `statement`, + `contract`, `photo`, `warranty`, `import_source`, `notices`, + `correspondence`, `plans`, `history`, `other`. + - `duplicate_of` names the attachment a withdrawn duplicate stands for + (`attachment.withdrawn`'s optional `duplicate_of`, schema v16). - Documents filed via `books.file_document` and receipts via `books.attach_receipt` share the blob store; they differ only in metadata. - **Receipts change by events, never by edits.** `receipt.unlinked` (`attachment_id`, `entry_id`, `reason`) detaches a receipt from its entry; - `attachment.withdrawn` (`attachment_id`, `reason`) retires an unlinked - receipt or a document (`withdrawn_at`, `withdrawn_reason` columns, schema - v5). Linking requires an unlinked, non-withdrawn attachment, so evidence + `attachment.withdrawn` (`attachment_id`, `reason`, `duplicate_of?`) + retires an unlinked receipt or a document (`withdrawn_at`, + `withdrawn_reason`, `duplicate_of` columns, schema v5 and v16). Linking requires an unlinked, non-withdrawn attachment, so evidence only leaves an entry through a recorded unlink. Blobs are untouched: a withdrawn file is still hash-verified and readable, and the journal shows when it was filed, linked, unlinked, and withdrawn, and why. +- **Extraction is derived, and journaled** (P6, rules.md B-44): + `attachment.text_extracted {attachment_id, text_sha256, extractor, + version, pages, status}` records what reading a file found (`ok`, + `none`, `failed`); the text itself is a content-addressed blob like any + other, and `books.verify` checks its presence and hash, never re-derives + it. One record per attachment, extractor and version: an extractor + upgrade adds a record, never replaces one. +- **Annotations and settlements are records, not edits** (P6): + `attachment.annotated {attachment_id, fields, note?}` is what a human + read off a file (the closed field list; a later annotation replaces the + earlier record), and `entry.settles {entry_id, settles[]}` says which + entries a payment pays back (the only source of "reimbursed"). - The store is policy-free: it records attachments faithfully. Whether a posting *requires* a receipt is decided by the MCP layer's receipt policy (`docs/mcp-interface.md`), which reads the `attachments` table. @@ -168,7 +185,11 @@ Suspected corruption is fixed by rebuild, never by forensics. `integrity_check` on startup. - **Schema.** `meta(key, value)` — `schema_version`, `base_currency`, `last_applied_seq`; `accounts`, `entries`, `lines`, `blobs`, `attachments` - as above; `events(seq, type, payload, created_at)` — the applier's output, + as above; `extractions(attachment_id, extractor, version, text_sha256, + pages, status, seq)` and `annotations(attachment_id, receipt_number, + payee, amount, date, instrument, category, source, reimbursable_to, + note, seq)` (P6; amounts TEXT, never REAL); `events(seq, type, payload, + created_at)` — the applier's output, a queryable mirror of the journal stream and the feed the future GUI tails. (Poll `WHERE seq > ?`; later SSE over the HTTP transport. This is the correct answer to "watch its files to update live" — watch the event diff --git a/docs/spec/rules.md b/docs/spec/rules.md index 147bebe..9e6c78e 100644 --- a/docs/spec/rules.md +++ b/docs/spec/rules.md @@ -1753,7 +1753,11 @@ that moves no figure); S6's payments with `line`, `discount`, `invoice.written_off`. An older version would restore the document without them silently: every invoice line would post to the revenue role and every bill line to general expenses, and every new -event type would be refused. A command asks for +event type would be refused. P6's is `attachment_text` (an +`attachment.text_extracted`, `attachment.annotated` or `entry.settles`, a +new event type, and an `attachment.withdrawn` carrying `duplicate_of`; an +older version would refuse the types, or withdraw a duplicate without +saying what it duplicates, B-44). A command asks for the feature it needs (`Store::ensure_feature`) before its first journaled line, and only when the line it is about to write uses it, so a write refused for waiting on the upgrade journals nothing, never an entry @@ -3316,3 +3320,79 @@ check_discount}`, `arap::{Application, Fee, SettleParams, SettlementRow}`, `ledger-cli` `tests/documents.rs`; `ledger-package`'s aging tie-out test. The Lean step is `Documents.Step.document` with every control line tagged (`Documents.settle_unattached`); no new `Step` constructor. + +## B-44 Evidence is read from the bytes, never trusted over them; identical files are kept once + +```text +GOAL: a receipt's text and the facts read off it are derived records the + books can re-make; the file's bytes stay the authority; the same + bytes are one file wherever they are evidence; and a payment says + what it settles instead of the books guessing. +SOURCE: Rev. Proc. 98-25 §5.01 (machine-sensible records with an audit + trail: the relationship between the records and the books must be + demonstrable); IRS Pub. 583 (supporting documents are kept with + the entries they support); plan/served.md P6 (defects 4 and 10); + IZ-18 (no FIFO guess); + CONVENTION (the extractor's version is part of its record; a + failed read is recorded, not hidden; a duplicate is withdrawn, not + deleted). +INVARIANTS: + 1. An attachment's text is a record (`attachment.text_extracted`) with + the text's content-addressed blob, the extractor and its version, the + page count and `ok`, `none` or `failed`. One record per attachment, + extractor and version: an upgrade adds a record and NEVER replaces + one, and the bytes are never re-derived by `verify` (its hash is + checked). + 2. Extraction is derived and journaled, never authoritative: a report, + a posting or an audit reads the file's bytes when the text matters, + and a failed extraction is a record that a human must read the + pages (D-15: no OCR). + 3. Identical bytes are one file: filing bytes already stored (and not + withdrawn) returns the existing attachment, linked to the new + subject. Consolidating legacy duplicates keeps the oldest, links it + to every subject the others had, and withdraws the others with + `duplicate_of`; nothing is deleted and no posted entry the receipt + policy covers is left without evidence. + 4. An annotation (`attachment.annotated`) records what a human read off + a file and changes no amount, no file and no posting; its fields are + a closed list. At least one field or the note is present. + 5. `entry.settles` names the entries a payment pays back, every named + entry existing, none itself, none twice. "Reimbursed" is derived from + it alone: where no settlement is recorded, Needs attention asks; the + books never guess (IZ-18). + 6. Every refusal leaves the journal unchanged; NEVER panics. +SCENARIOS: + 1. A text PDF filed: one extraction record, its text a blob, `ok`. + 2. The same PDF filed again: one attachment, linked to the second + subject, `deduplicated: true`. + 3. Two legacy records with one blob, the oldest withdrawn, the youngest + an entry's only receipt: consolidation is refused whole. + 4. The same, both live: the oldest is linked to the other's subjects, + the other is unlinked and withdrawn with `duplicate_of`. + 5. A receipt annotated 7 on two different files: the next engine number + is 8 (numbers run from the max; a human's duplicate is not renumbered). + 6. An amount 20.00 paid by a member, reimbursed: the reimbursement's + `entry.settles` names the expense; no Needs-attention item remains. + 7. The same with no settlement: Needs attention asks, once. +``` + +**Enforced:** `ledger_import::extract` (`pdf_text`, `pdf_lines`, +`page_png`, `xlsx_text`, `text_of`, `extract`; caps 25 MiB, 30 pages, +2 MiB of text; a reader panic is caught), `ledger_core::evidence::{Annotation, +AnnotationFields}`, `Ledger::{annotate_attachment, next_receipt_number, +entry_settles, settles_entry, set_entry_settles}`, +`Store::{extract_attachment, annotate_attachment, set_entry_settles, +withdraw_attachment_duplicate}` and `Store::attach`'s dedup; feature +`attachment_text`. +**Checked:** `ledger-import` `tests/extract.rs` +(`extraction_deterministic`, `xlsx_cells_are_decimal_strings`, +`a_pdf_that_crashes_the_reader_is_failed_not_fatal`); `ledger-store` +`tests/attachments.rs` (`extraction_is_journaled_once_per_version`, +`extract_waits_for_attachment_text`, `same_bytes_one_record_many_links`, +`legacy_duplicates_replay_as_written`, +`annotations_replay_and_project`, `receipt_numbers_run_from_the_max`, +`reimbursed_is_derived_from_settles_only`); `ledger-cli` +`tests/evidence.rs` (`get_receipt_default_has_text_and_no_bytes`, +`page_image_is_a_png_of_the_page`, +`consolidation_never_removes_an_entrys_only_evidence`); fuzz targets +`fuzz_extract_pdf` and `fuzz_extract_xlsx`. diff --git a/docs/testing.md b/docs/testing.md index 3360ee9..8f0d5b5 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -3269,3 +3269,111 @@ the blob-before-journal order, the `File`/`Existing` link steps) and the feature gate; the MCP handler's amount default, the exactly-one evidence check, and the preview's clone; attention's year comparison and its `stands_on` filter; the bookspec loader's `may_refuse` arm. + +## P6: the evidence pipeline (2026-10-27) + +Extraction, dedup, annotations and company record categories (plan P6; +rules.md B-44). One in-process extractor in `ledger-import` +(`src/extract.rs`): `pdf_text` and `pdf_lines` (promoted from the PDF +reader test), `page_count`, `page_png` (promoted from the viewer's private +rasterizer), `xlsx_text` (cells as the exact decimal strings the sheet XML +holds), `text_of` (moved from `tin_files`, now reading a PDF's text too) +and `extract`, which reports `ok`, `none` or `failed` under the caps (25 +MiB in, 30 pages, 2 MiB of text). A reader panic is caught there, never +reaching the server (D-32: no sandbox, and the plan says so). The viewer +and the server's filing scan both read through it: `ledger-gui/src/receipts.rs` +keeps no `hayro` use of its own. + +- **ledger-import:** `tests/extract.rs` `extraction_deterministic` (same + bytes, same text and PNG bytes; plain text and an image's `none`), + `xlsx_cells_are_decimal_strings` (proptest: the cell text is the decimal + string written, never a float), `a_pdf_that_crashes_the_reader_is_failed_not_fatal` + (garbage and truncated PDFs are `Err`, the too-large cap is refused + before the reader, page 31 and a zero-size page are refused); unit tests + in `src/extract.rs` for the panic catch and the input cap. `tests/spec_import.rs` + keeps its script/RTL PDF test; its `pdf_lines` helper is now the + extractor's. +- **ledger-core:** `src/evidence.rs` (`Annotation`, `AnnotationFields`; + `checked`), `Ledger::{annotate_attachment, next_receipt_number, + entry_settles, settles_entry, set_entry_settles}`, and `differs_from` + gains both. No kernel or Lean change: no money moves. +- **ledger-store:** feature `attachment_text` (format gate; + `attachment_text_is_its_own_feature`), the three event types + (`format.rs`, `meta.rs` readers, replay and projection arms), schema v16 + (`extractions`, `annotations`, `attachments.duplicate_of`). `attach` + returns the live record for bytes already stored (linked to the new + subject), `extract_attachment`, `annotate_attachment`, + `set_entry_settles`, `withdraw_attachment_duplicate`, + `attachments_holding_text` and `attachments_with_receipt_number`; + `verify` checks each extraction's text blob hash and never re-derives it; + `stored_blob_shas` includes it. `tests/attachments.rs`: + `extraction_is_journaled_once_per_version`, + `extract_waits_for_attachment_text` (R-A: refused before the upgrade, + journaling nothing; accepted after), `same_bytes_one_record_many_links` + (proptest, eight cases: one record, one extraction, a link per filing), + `legacy_duplicates_replay_as_written`, `annotations_replay_and_project`, + `receipt_numbers_run_from_the_max`, + `reimbursed_is_derived_from_settles_only`. The `p6_evidence` fixture + (`tests/fixtures/old-books/`) was written by the version before the unit + and opens to the same figures. +- **ledger-mcp:** `tools/evidence.rs` (`books.find_attachment`, + `books.annotate_attachment`, `books.set_entry_settles`, + `books.consolidate_duplicates`), `books.get_receipt`'s `text` and + `page_image` views (meta carries the first 8 KiB and the page count), + `books.file_document`'s four new categories, the new store and snapshot + methods, and the resource reads `books://attachments/{id}` and + `books://attachments/{id}/page/{n}` (a PNG blob). +- **ledger-package / ledger-gui:** Needs attention gains "extraction + failed" and "member-paid entry with no settlement recorded" (R-F) over + `PackageBooks::extractions`; the viewer's Documents page groups by + category and filters by subject; `Books` carries the extractions. +- **ledger-cli:** `tests/evidence.rs` `get_receipt_default_has_text_and_no_bytes` + (the default is under 8 KiB, has the text and no file; the resource + matches), `page_image_is_a_png_of_the_page` (IHDR-checked, page 2 + refused, `bytes` unchanged, the page resource a PNG blob), + `annotations_and_searches` (the closed fields, the engine number, the + three searches, a settlement, exactly-one-selector), + `consolidation_never_removes_an_entrys_only_evidence` (a hand-written + legacy pair whose kept record is withdrawn: `receipt_required`, journal + unchanged) and `consolidation_links_the_kept_file_to_every_subject` + (preview journals nothing; the kept file takes the subject; the other is + withdrawn with `duplicate_of`; verify ok), + `every_p2_served_attachment_has_a_small_default_answer` (all 64 of the + fixture's attachments under 8 KiB), and + `the_backfill_reads_an_old_fixtures_files_once` (the `p6_evidence` + fixture upgraded, read through the admin command and then through + `wanderbooks books extract`, a second run journaling nothing). The + cutover test `books_p2_served_cut_over_whole` gained the annotation + call: refused before consent, accepted after. +- **fuzz:** `fuzz_extract_pdf` and `fuzz_extract_xlsx` in + `fuzz/fuzz_targets/`, in the scheduled workflow's matrix. + +**Done when, checked:** (1) the two fixture tests above; (2) no `hayro` +use in `ledger-gui` or `ledger-mcp` (the extract crate is the only one); +(3) the dedup property and the only-evidence refusal pass; (4) the +backfill command test; (5) the PR carries the 8.0.6 checklist; (6) +`just check` green and both fuzz targets run ten minutes clean. + +**For P12 (mutation targets, none run, D-33):** `ledger-import::extract`: +the caps' comparisons (`MAX_BYTES`, `MAX_PAGES`, `MAX_TEXT_BYTES`, the +`cap_text` boundary walk), `extract`'s order (PDF by type or magic, XLSX +by type or extension, then plain text) and `found`'s blank-text arm; +`pdf_lines`' page `take`, the baseline rounding and the left-to-right +sort; `page_png`'s scale floor and its page bounds; `xlsx_text`'s shared +string index, the `t="s"`/`inlineStr`/`b` arms and the sheet-name filter. +`ledger-core::evidence`: `checked`'s trims, the zero number, the negative +amount and the empty-annotation refusal; `next_receipt_number`'s +`saturating_add`. `ledger-store`: `attach`'s live-record lookup and link +arms, `extract_and_journal`'s text/blob branch and status mapping, +`extraction_for`'s triple (attachment, extractor, version), +`withdraw_attachment_as`'s `duplicate_of` arm and feature check, the +`verify` text-blob loop, and the projection arms' conflict keys. +`ledger-mcp::evidence`: `find_attachment`'s exactly-one selector, the +hex check and the text length floor; `consolidate_duplicates`' grouping +(`< 2`), the keep/dups split, the evidence `left == 0` gate, the keep +`withdrawn` test, the unlink dedup and the `withdrawn` count; the +annotation amount/date parse and the next number; `get_receipt`'s view +arms, the 8 KiB cut, the 64 KiB hand-over threshold and `page - 1`. +`ledger-package::attention`: the extraction `seq` comparison and the +`Failed` filter, the owed-account name test, the credit side and the +`settles_entry` skip. diff --git a/docs/verification.md b/docs/verification.md index 361405e..7fb091a 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -876,3 +876,17 @@ each harness's whole run, its compile of about 50 s included): | `bounds_kept_is_the_definition_on_books_in_bounds` | 34 | prove | (as above) | | `settle_is_exact` | 36 | prove | 1 | | `kani_symbolic_copy_canary` | (the checker: "Trusting the checker") | prove | 3 | + + +## P6: the evidence pipeline + +No kernel arithmetic, `Ledger` command in the formal sense, or Lean step +changed (P6 moves no money): `just prove` is unchanged and stays green. +The new `Ledger` methods (`annotate_attachment`, `set_entry_settles`) are +replay-side records with validation only, covered by +`annotations_replay_and_project` and `reimbursed_is_derived_from_settles_only`; +the arithmetic-shaped work (the extraction caps and the page scale) is in +`ledger-import`, outside the kernel, and is tested and fuzzed rather than +proved. The formal bar for the next unit that touches these paths is +unchanged; the P12 paragraph in `docs/testing.md` lists the mutation +targets. diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 0a76f4a..381a730 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -78,6 +78,20 @@ test = false doc = false bench = false +[[bin]] +name = "fuzz_extract_pdf" +path = "fuzz_targets/fuzz_extract_pdf.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "fuzz_extract_xlsx" +path = "fuzz_targets/fuzz_extract_xlsx.rs" +test = false +doc = false +bench = false + [[bin]] name = "fuzz_config_parse" path = "fuzz_targets/fuzz_config_parse.rs" diff --git a/fuzz/fuzz_targets/fuzz_extract_pdf.rs b/fuzz/fuzz_targets/fuzz_extract_pdf.rs new file mode 100644 index 0000000..602b582 --- /dev/null +++ b/fuzz/fuzz_targets/fuzz_extract_pdf.rs @@ -0,0 +1,17 @@ +//! The PDF extractor on arbitrary bytes (plan P6): reading text, a page's +//! pixels and the page count is a refusal or an answer, never a panic or a +//! hang (D-32: it runs in the server's process). +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + let _ = ledger_import::extract::pdf_text(data); + let _ = ledger_import::extract::pdf_lines(data); + if let Ok(pages) = ledger_import::extract::page_count(data) { + for page in [0, pages.saturating_sub(1), pages] { + let _ = ledger_import::extract::page_png(data, page, 64); + } + } + let _ = ledger_import::extract::text_of(data, "application/pdf", "receipt.pdf"); +}); diff --git a/fuzz/fuzz_targets/fuzz_extract_xlsx.rs b/fuzz/fuzz_targets/fuzz_extract_xlsx.rs new file mode 100644 index 0000000..f0263c7 --- /dev/null +++ b/fuzz/fuzz_targets/fuzz_extract_xlsx.rs @@ -0,0 +1,15 @@ +//! The XLSX extractor on arbitrary bytes (plan P6): a zip or not, a sheet +//! or not, it is a value or a refusal, never a panic; a cell's number is +//! read as the decimal string the XML holds. +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + let _ = ledger_import::extract::xlsx_text(data); + let _ = ledger_import::extract::text_of( + data, + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "sheet.xlsx", + ); +}); diff --git a/fuzz/fuzz_targets/fuzz_store_replay.rs b/fuzz/fuzz_targets/fuzz_store_replay.rs index a1425fb..97f933f 100644 --- a/fuzz/fuzz_targets/fuzz_store_replay.rs +++ b/fuzz/fuzz_targets/fuzz_store_replay.rs @@ -22,27 +22,29 @@ use libfuzzer_sys::fuzz_target; use serde_json::{Map, Value, json}; const TYPES: &[&str] = &[ - "account.added", "account.renamed", "account.tax_set", "asset.depreciation_posted", - "asset.disposed", - "asset.registered", "attachment.withdrawn", "bill.created", "bill.paid", "class.added", - "customer.added", "document.filed", "entry.created", "entry.line_added", - "entry.line_class_set", "entry.posted", "entry.reversed", "entry.vendor_set", - "fx.rate_set", "invoice.created", "invoice.payment_received", "invoice.sent", - "invoice.written_off", - "journal.tail_repaired", - "line.cleared", "payroll.run", "period.closed", "period.created", "receipt.attached", - "receipt.linked", "receipt.unlinked", "recurring.generated", "recurring.registered", - "schedule.posted", "schedule.registered", "vendor.1099_set", "vendor.added", - "journal.format_set", "settings.set", "convention.set", "convention.retired", - "decision.recorded", "open_item.raised", "open_item.resolved", "entry.approved", - "vendor.w9_flag_set", "instrument.added", "entry.instrument_set", "asset.basis_set", - "party.creditor_set", "account.renumbered", "asset.acquisition_linked", "asset.corrected", - "asset.described", "asset.serial_set", "asset.tagged", "asset.tax_basis_set", - "attachment.linked", "attachment.unlinked", "entry.no_party", "entry.parties_set", - "entry.party_set", "party.added", "party.role_set", "party.tin_set", "party.updated", - "party.w4_set", "party.w9_set", "payroll.taxes_remitted", "reconciliation.completed", - "reconciliation.voided", "role.assigned", "statement.recorded", "statement.voided", - "project.set", + "account.added", "account.renamed", "account.tax_set", "asset.depreciation_posted", + "asset.disposed", + "asset.registered", "attachment.annotated", "attachment.text_extracted", "attachment.withdrawn", + "bill.created", "bill.paid", "class.added", + "customer.added", "document.filed", "entry.created", "entry.line_added", + "entry.line_class_set", "entry.posted", "entry.reversed", "entry.vendor_set", + "fx.rate_set", "invoice.created", "invoice.payment_received", "invoice.sent", + "invoice.written_off", + "journal.tail_repaired", + "line.cleared", "payroll.run", "period.closed", "period.created", "receipt.attached", + "receipt.linked", "receipt.unlinked", "recurring.generated", "recurring.registered", + "schedule.posted", "schedule.registered", "vendor.1099_set", "vendor.added", + "journal.format_set", "settings.set", "convention.set", "convention.retired", + "decision.recorded", "open_item.raised", "open_item.resolved", "entry.approved", + "vendor.w9_flag_set", "instrument.added", "entry.instrument_set", "asset.basis_set", + "party.creditor_set", "account.renumbered", "asset.acquisition_linked", "asset.corrected", + "asset.described", "asset.serial_set", "asset.tagged", "asset.tax_basis_set", + "attachment.linked", "attachment.unlinked", "entry.no_party", "entry.parties_set", + "entry.party_set", "party.added", "party.role_set", "party.tin_set", "party.updated", + "party.w4_set", "party.w9_set", "payroll.taxes_remitted", "reconciliation.completed", + "reconciliation.voided", "role.assigned", "statement.recorded", "statement.voided", + "project.set", "entry.settles", + // Merged 2026-10-28 (S7, P6): every type EVENT_TYPES names. ]; const FIELDS: &[&str] = &[ diff --git a/plan/served.md b/plan/served.md index 25b252f..f17a907 100644 --- a/plan/served.md +++ b/plan/served.md @@ -1475,7 +1475,16 @@ Each unit follows the 8.0.4 template. "A-n" is an owner answer numbered in 8.3.1 5. **Files:** owns `crates/ledger-import/src/extract.rs`, `crates/ledger-mcp/src/tools/evidence.rs`, the two fuzz targets; touches `crates/ledger-gui/src/receipts.rs`, `crates/ledger-mcp/src/tin_files.rs`, `store.rs` (`attach`), `crates/ledger-host/src/commands.rs`, the 8.0.3 files. 6. **Tests and proofs:** named above; no kernel or Lean change (no money moves). 7. **Done when:** (1) every attachment of the `p2-served` fixture and of the demo books gives a `get_receipt` default answer under 8 KiB; (2) the viewer and the TIN scan read PDFs through the one `extract` module (no `hayro` use left in `ledger-gui` beyond the call, none in `ledger-mcp`); (3) the dedup property and the only-evidence refusal pass; (4) the backfill command runs on a fixture and is idempotent; (5) 8.0.6 in the PR; (6) `just check` green and the two fuzz targets run clean for 10 minutes locally. -8. **As built:** none yet. +8. **As built** (2026-10-27, branch `P6`): every step built; departures and findings: + - Step 1: `ledger-import/src/extract.rs` also promotes `pdf_lines` (the per-line glyphs its tests read), adds `page_count` and `extract` (the `ok`/`none`/`failed` answer the journal records), and `extract::VERSION` is the crate's version. The renderer `hayro` moves to `ledger-import`'s dependencies as well as `hayro-interpret`: `page_png` rasterizes, and the renderer is not in `hayro-interpret`. Caps are constants (`MAX_BYTES`, `MAX_PAGES`, `MAX_TEXT_BYTES`). + - Step 2: `ledger-store` depends on `ledger-import`, so the writer reads a file as it is filed and on the backfill. Attach-time extraction runs only in books that have declared `attachment_text`: books waiting for the owner's go-ahead keep filing (an existing store test requires it), and `wanderbooks books extract` reads their old and new files after the upgrade. The command is `wanderbooks books extract BOOK ATTACHMENT` (the plan's `extract ID` needed the book as the other `books` commands name it). + - Step 3: `books.get_receipt` takes `page` and `max_px` as flat arguments beside `view` (the MCP arguments are one object; the plan's `page_image { page, max_px }` reads as those two), and its meta carries `text_status` too. The page resource is a PNG blob (`BlobResourceContents`), the first binary resource; `books://receipts/{id}` stays an alias of `books://attachments/{id}`. + - Step 4: consolidation groups every record sharing a blob, withdrawn ones included, so the only-evidence refusal is reachable (a withdrawn oldest record cannot stand in); a preview changes nothing and is served on books that haven't declared the feature, the confirmed call refuses before any journal line. + - Step 5: `books.set_entry_settles` is the Build step's name but is not in 8.0.3's new-tools row; the Build step's name is used (flagged to the EM). Annotations are ledger state (`ledger_core::evidence`) as well as a projection table, because `Ledger::next_receipt_number` reads them; `entry.settles` is ledger state only, and `differs_from` compares both. + - `books.find_attachment`'s `sha256` is exempt from the D-46 inline-argument refusal (it names a search key, not bytes), and `no_tool_takes_a_path` exempts an annotation's `source` (what a human says a record came from, never a place). + - Two `ledger-store/tests/spec_store.rs` tests encoded the invariant step 4 replaces (two records for one blob); they are rewritten as `same_bytes_are_one_attachment_linked_to_both` and `the_first_filings_name_and_type_stay`. + - Schema 16 (8.0.3's provisional 20 was a guide; numbers are taken at merge). No kernel, Kani or Lean change: no money moves, so `just prove` is unchanged. + - 8.0.6: `tests/store.rs::books_written_under_older_rules_still_open` gained the three new event shapes; the fixture `tests/fixtures/old-books/p6_evidence` was written by the version before the unit (its API, its `expected.json`); the cutover test's refused list gained `books.annotate_attachment`; `attachment_text_is_its_own_feature` and the `tests/format.rs` arms pass; no figure moves, so no golden changed; merging needs no go-ahead (the live books refuse the new writes until the owner gives it). **Struck from P6:** ~~`ledger-extract` sandboxed subprocess (rlimits, seccomp, Landlock), `extractor_sandboxed`~~ → D-32, D-53; ~~`lopdf`~~ → hayro; ~~`attach_email`, `attachment.rendered`, `fuzz_eml`~~ → D-17 (email-in waits for the hosted service; a receipt that is an email is printed to PDF by the human, as the first company's convention says); ~~`compare_reference`, `record_inventory`, `check_inventory`, `inventory.recorded`, `reference.compared`~~ → the Sheet and the books as of 2026-10-11 are frozen and the final `verify-sheet` run is a filed workpaper; P10 step 5 cites it (IZ-3b, Z-6); ~~`inline_base64_requires_sha`~~ → D-46; ~~`reimbursed` by FIFO~~ → `entry.settles` only; ~~"the books dir's `company/` folder retired"~~ → done at the P1 and P2 cutovers.