From 3a9aa92315a5e80fe310c229936f6f31f3e57c42 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Tue, 20 Oct 2026 19:59:32 -0600 Subject: [PATCH] P3: the books' format moves only with the owner's go-ahead (D-39) Books an update could write a newer format for open on their old format and are asked at every open; anything needing a new feature is refused until the owner agrees with the passphrase, after a verified backup that includes the keys. New books start on the current format. Each projection schema has its own database file, so an older version still opens the books until the owner agrees. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01T3gXMVt7XJPi6ens1or9Dj --- Cargo.lock | 2 + crates/ledger-cli/Cargo.toml | 1 + crates/ledger-cli/src/main.rs | 13 +- crates/ledger-cli/tests/format_upgrade.rs | 612 ++++++++++++++++++ crates/ledger-cli/tests/open_while_held.rs | 1 + crates/ledger-gui/Cargo.toml | 1 + crates/ledger-gui/src/app.rs | 3 + crates/ledger-gui/src/books.rs | 4 + crates/ledger-gui/src/owner.rs | 252 +++++++- crates/ledger-gui/tests/format_upgrade.rs | 396 ++++++++++++ crates/ledger-gui/tests/newer_journal.rs | 9 +- crates/ledger-gui/tests/old_books.rs | 5 +- crates/ledger-gui/tests/owner_menu.rs | 4 + crates/ledger-gui/tests/parties.rs | 17 +- crates/ledger-host/src/commands.rs | 449 ++++++++++++- crates/ledger-host/src/host.rs | 95 ++- crates/ledger-host/src/lease.rs | 32 +- crates/ledger-mcp/handbook/format-upgrade.md | 7 + crates/ledger-mcp/src/admin.rs | 175 ++++- crates/ledger-mcp/src/backup.rs | 202 +++++- crates/ledger-mcp/src/handbook.rs | 4 + crates/ledger-mcp/src/lib.rs | 5 +- crates/ledger-mcp/src/local.rs | 411 ++++++++++++ crates/ledger-mcp/src/server.rs | 75 ++- crates/ledger-mcp/src/tools/kept.rs | 38 +- crates/ledger-mcp/src/types.rs | 10 +- crates/ledger-mcp/tests/mcp_stdio.rs | 32 +- crates/ledger-mcp/tests/spec_1099.rs | 88 ++- crates/ledger-mcp/tests/spec_mcp.rs | 2 +- crates/ledger-mcp/tests/spec_party_totals.rs | 89 ++- crates/ledger-mcp/tests/spec_vault.rs | 4 +- crates/ledger-store/src/backup.rs | 137 ++++ crates/ledger-store/src/error.rs | 30 + crates/ledger-store/src/format.rs | 281 +++++++- crates/ledger-store/src/lib.rs | 6 +- crates/ledger-store/src/root.rs | 23 +- crates/ledger-store/src/snapshot.rs | 7 +- crates/ledger-store/src/store.rs | 389 ++++++++--- crates/ledger-store/tests/asset_orders.rs | 8 +- crates/ledger-store/tests/asset_tags.rs | 65 +- crates/ledger-store/tests/assets.rs | 37 +- crates/ledger-store/tests/attachments.rs | 4 +- .../tests/fixtures/p2-books/journal.jsonl | 7 + .../tests/fixtures/p2-books/wanderbooks.db | Bin 0 -> 212992 bytes crates/ledger-store/tests/form1099.rs | 144 +++-- crates/ledger-store/tests/format.rs | 197 +++--- crates/ledger-store/tests/format_upgrade.rs | 390 +++++++++++ crates/ledger-store/tests/member_closes.rs | 105 ++- crates/ledger-store/tests/parties.rs | 4 +- crates/ledger-store/tests/party_creditors.rs | 147 +++-- crates/ledger-store/tests/party_tin.rs | 4 +- crates/ledger-store/tests/root.rs | 37 +- crates/ledger-store/tests/schema_files.rs | 371 +++++++++++ crates/ledger-store/tests/settings.rs | 49 +- crates/ledger-store/tests/snapshot.rs | 2 +- crates/ledger-store/tests/spec_store.rs | 22 +- crates/ledger-store/tests/store.rs | 88 +-- docs/mcp-interface.md | 63 +- docs/receipt-store.md | 8 +- docs/spec/rules.md | 141 +++- docs/testing.md | 122 +++- plan/served.md | 11 +- 62 files changed, 5309 insertions(+), 628 deletions(-) create mode 100644 crates/ledger-cli/tests/format_upgrade.rs create mode 100644 crates/ledger-gui/tests/format_upgrade.rs create mode 100644 crates/ledger-mcp/handbook/format-upgrade.md create mode 100644 crates/ledger-store/tests/fixtures/p2-books/journal.jsonl create mode 100644 crates/ledger-store/tests/fixtures/p2-books/wanderbooks.db create mode 100644 crates/ledger-store/tests/format_upgrade.rs create mode 100644 crates/ledger-store/tests/schema_files.rs diff --git a/Cargo.lock b/Cargo.lock index 8f82173..4523853 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3831,6 +3831,7 @@ name = "ledger-cli" version = "0.1.0" dependencies = [ "base64 0.22.1", + "chrono", "ledger-core", "ledger-gui", "ledger-host", @@ -3902,6 +3903,7 @@ dependencies = [ "serde", "serde_json", "tempfile", + "zeroize", "zip 9.0.2", ] diff --git a/crates/ledger-cli/Cargo.toml b/crates/ledger-cli/Cargo.toml index 92a0610..f320002 100644 --- a/crates/ledger-cli/Cargo.toml +++ b/crates/ledger-cli/Cargo.toml @@ -24,6 +24,7 @@ ledger-gui = { path = "../ledger-gui", optional = true } [dev-dependencies] base64 = "0.22" +chrono = "0.4" libc = "0.2" reqwest = "0.13" rmcp = { version = "3", features = ["client", "transport-streamable-http-client-reqwest"] } diff --git a/crates/ledger-cli/src/main.rs b/crates/ledger-cli/src/main.rs index 5e00d48..42048ce 100644 --- a/crates/ledger-cli/src/main.rs +++ b/crates/ledger-cli/src/main.rs @@ -16,7 +16,7 @@ fn print_usage() { eprintln!("usage: wanderbooks setup [--root PATH] [--home DIR]"); eprintln!(" wanderbooks gui [BOOK | --root PATH --book ID | --data-dir PATH ...]"); eprintln!( - " wanderbooks books list | create ID | adopt DATA-DIR --id ID | restore BACKUP | back-up ID | put-aside ID DATA-DIR [PATH...]" + " wanderbooks books list | create ID | adopt DATA-DIR --id ID | restore BACKUP | back-up ID | upgrade ID | put-aside ID DATA-DIR [PATH...]" ); eprintln!(" wanderbooks connect claude|codex --book ID"); eprintln!( @@ -31,7 +31,9 @@ fn print_usage() { eprintln!(); eprintln!(" setup make the books' private folder (only your account can open it)"); eprintln!(" gui open the live, read-only view of the books (a book of the data root)"); - eprintln!(" books list, start, adopt, restore or back up books (`books --help`)"); + eprintln!( + " books list, start, adopt, restore, back up or upgrade books (`books --help`)" + ); eprintln!(" connect let Claude Code or Codex reach the books"); eprintln!(" token make, list or revoke an agent's token for books (`token --help`)"); eprintln!(" mcp an agent's way to the books, over stdio (what `connect` registers)"); @@ -112,6 +114,10 @@ impl ledger_gui::owner::Owner for ViewerOwner { }; ledger_host::commands::connect_agent(&self.0, agent, book) } + + fn upgrade_format(&self, book: &str, passphrase: &str) -> Result, String> { + ledger_host::commands::upgrade_format(&self.0, book, passphrase) + } } /// `wanderbooks gui`: the viewer, in this process. For books of the data @@ -142,9 +148,10 @@ fn run_gui(args: Vec) -> i32 { let root = value("--root").map(PathBuf::from); let book = value("--book"); if known && ledger_gui::book_in_root(root.as_deref(), book.as_deref()).is_ok() { + // The viewer asks about a format upgrade in its own window. let common = ledger_host::commands::Common { root, - knobs: Default::default(), + ..Default::default() }; ledger_gui::owner::set_owner(std::sync::Arc::new(ViewerOwner(common.clone()))); ledger_host::commands::hold_for_viewer(&common, book); diff --git a/crates/ledger-cli/tests/format_upgrade.rs b/crates/ledger-cli/tests/format_upgrade.rs new file mode 100644 index 0000000..993a74b --- /dev/null +++ b/crates/ledger-cli/tests/format_upgrade.rs @@ -0,0 +1,612 @@ +//! The format upgrade from end to end (plan/served.md 3.9, D-39), through +//! the one executable: books an older wanderbooks wrote, in a data root, +//! opened by the books server of this version. Oracle: the plan's +//! `format_upgrade_waits_for_consent` (they open on the old format until +//! the owner consents; a backup with the keys is made first; no MCP tool +//! can consent; declining leaves them openable) and D-39 (asked at every +//! open until given; books.start puts it first). +//! +//! ```text +//! INVARIANTS: +//! 1. Waiting books open and are served; books.start says so first, and +//! a change that needs the new format is refused, journaling nothing; +//! a change that needs nothing new is made. +//! 2. No tool consents: none is offered, and none moves the format. +//! 3. The owner's commands ask at every open, at a terminal; elsewhere +//! they say so. Declining, or a wrong passphrase, changes nothing. +//! 4. With the passphrase: a backup of the books as they were, keys in +//! it, then the whole format in one line; the agent's session goes on +//! and the change is made; the books verify, and reopen upgraded. +//! ``` +#![cfg(target_os = "linux")] + +use serde_json::{Value, json}; +use std::io::{BufRead, BufReader, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::sync::mpsc; +use std::time::{Duration, Instant}; + +/// Short timings for tests, and agents' tokens kept in files in the test +/// root, not this computer's keyring. +const KNOBS: &str = "close_after=2500,backoff=50,exe_poll=100,init_wait=5000,queue_wait=30000,file_keyring=1,backup_quiet=300"; + +/// The owner passphrase the tests set. +const PASS: &str = "correct horse battery"; + +/// A home with a data root (`setup`) and its owner passphrase. +struct Home { + dir: tempfile::TempDir, +} + +impl Home { + fn new(tag: &str) -> Self { + let dir = tempfile::Builder::new() + .prefix(&format!("wb-upgrade-{tag}-")) + .tempdir() + .unwrap(); + let out = Command::new(env!("CARGO_BIN_EXE_wanderbooks")) + .args(["setup", "--home"]) + .arg(dir.path()) + .env_remove("XDG_DATA_HOME") + .stdin(Stdio::null()) + .output() + .unwrap(); + assert!(out.status.success(), "{out:?}"); + let home = Self { dir }; + ledger_store::passphrase::set(&home.root().join("auth"), PASS, None).unwrap(); + home + } + + fn root(&self) -> PathBuf { + self.dir.path().join(".local/share/wanderbooks") + } + + fn book_dir(&self, id: &str) -> PathBuf { + self.root().join("books").join(id) + } + + fn run_dir(&self) -> PathBuf { + self.root().join("run") + } + + /// The words of `mode` with the root and the test knobs. + fn words(&self, mode: &str) -> Vec { + vec![ + mode.to_string(), + "--root".into(), + self.root().display().to_string(), + "--test-knobs".into(), + KNOBS.into(), + ] + } + + /// The program, with the test home's environment alone. + fn cmd(&self, mode: &str) -> Command { + let mut c = Command::new(env!("CARGO_BIN_EXE_wanderbooks")); + c.args(self.words(mode)) + .env_clear() + .env("PATH", "/usr/bin:/bin") + .env("HOME", self.dir.path()); + c + } + + /// `books ARGS`, not at a terminal. + fn books(&self, args: &[&str]) -> std::process::Output { + let mut c = self.cmd("books"); + c.args(args).stdin(Stdio::null()); + c.output().unwrap() + } + + /// One line to the lease socket, the answer. + fn lease_line(&self, line: &str) -> String { + use std::io::Read; + let short = ledger_store::root::SocketPath::new(&self.run_dir(), "lease.sock").unwrap(); + let mut s = std::os::unix::net::UnixStream::connect(short.path()).unwrap(); + s.write_all(line.as_bytes()).unwrap(); + s.shutdown(std::net::Shutdown::Write).unwrap(); + let mut out = String::new(); + s.read_to_string(&mut out).unwrap(); + out + } + + /// The pids running `mode` for this root. + fn pids(&self, mode: &str) -> Vec { + let root = self.root().display().to_string(); + let mut out = Vec::new(); + for entry in std::fs::read_dir("/proc").unwrap().flatten() { + let Ok(pid) = entry.file_name().to_string_lossy().parse::() else { + continue; + }; + let Ok(raw) = std::fs::read(entry.path().join("cmdline")) else { + continue; + }; + let args: Vec = raw + .split(|b| *b == 0) + .map(|a| String::from_utf8_lossy(a).into_owned()) + .collect(); + if args.get(1).map(String::as_str) == Some(mode) && args.contains(&root) { + out.push(pid); + } + } + out + } + + /// `words` run at a terminal (a pseudo-terminal, through util-linux + /// `script`), typing `lines` one at a time, each once it has been + /// asked for (a prompt ending ": " or "] "): its exit code and what + /// the terminal showed. + fn at_terminal(&self, words: &[String], lines: &[&str]) -> (i32, String) { + let command = std::iter::once(env!("CARGO_BIN_EXE_wanderbooks").to_string()) + .chain(words.iter().cloned()) + .collect::>() + .join(" "); + let mut child = Command::new("script") + .args(["-qefc", &command, "/dev/null"]) + .env_clear() + .env("PATH", "/usr/bin:/bin") + .env("HOME", self.dir.path()) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("util-linux script"); + let mut stdin = child.stdin.take().unwrap(); + let shown = std::sync::Arc::new(std::sync::Mutex::new(Vec::::new())); + let mut stdout = child.stdout.take().unwrap(); + let reader = { + let shown = shown.clone(); + std::thread::spawn(move || { + use std::io::Read; + let mut buf = [0u8; 4096]; + while let Ok(n) = stdout.read(&mut buf) { + if n == 0 { + break; + } + shown.lock().unwrap().extend_from_slice(&buf[..n]); + } + }) + }; + let prompts = |shown: &[u8]| { + String::from_utf8_lossy(shown) + .split(['\r', '\n']) + .filter(|l| l.ends_with(": ") || l.ends_with("] ")) + .count() + }; + for (i, line) in lines.iter().enumerate() { + let deadline = Instant::now() + Duration::from_secs(120); + while prompts(&shown.lock().unwrap()) <= i { + if child.try_wait().unwrap().is_some() || Instant::now() > deadline { + break; + } + std::thread::sleep(Duration::from_millis(20)); + } + // It turns its echo off just after asking for a passphrase. + std::thread::sleep(Duration::from_millis(150)); + let _ = stdin.write_all(format!("{line}\n").as_bytes()); + } + let deadline = Instant::now() + Duration::from_secs(300); + while child.try_wait().unwrap().is_none() { + assert!(Instant::now() < deadline, "it never ended"); + std::thread::sleep(Duration::from_millis(50)); + } + drop(stdin); + let out = child.wait_with_output().unwrap(); + let _ = reader.join(); + let text = format!( + "{}{}", + String::from_utf8_lossy(&shown.lock().unwrap()), + String::from_utf8_lossy(&out.stderr) + ); + (out.status.code().unwrap_or(-1), text) + } +} + +impl Drop for Home { + fn drop(&mut self) { + let _ = self + .cmd("close") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let mut found = false; + for mode in ["serve", "mcp", "host", "server"] { + for pid in self.pids(mode) { + found = true; + // SAFETY: a process of this test's own root. + unsafe { libc::kill(pid as i32, libc::SIGKILL) }; + } + } + if !found || Instant::now() > deadline { + break; + } + std::thread::sleep(Duration::from_millis(100)); + } + } +} + +/// A child whose stdout lines arrive on a channel. +struct Running { + child: Child, + lines: mpsc::Receiver, +} + +impl Running { + fn start(mut c: Command) -> Self { + let mut child = c + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let (tx, lines) = mpsc::channel(); + let out = child.stdout.take().unwrap(); + std::thread::spawn(move || { + for line in BufReader::new(out).lines() { + let Ok(line) = line else { break }; + if tx.send(line).is_err() { + break; + } + } + }); + Self { child, lines } + } + + fn expect(&self, text: &str, secs: u64) -> String { + let deadline = Instant::now() + Duration::from_secs(secs); + loop { + let left = deadline.saturating_duration_since(Instant::now()); + match self.lines.recv_timeout(left) { + Ok(line) if line.contains(text) => return line, + Ok(_) => {} + Err(_) => panic!("no line with {text:?} in {secs} s"), + } + } + } +} + +impl Drop for Running { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +/// `serve`, holding the books open until dropped. +fn serve(home: &Home) -> Running { + let held = Running::start(home.cmd("serve")); + held.expect("The books are open", 60); + held +} + +/// An agent: the bridge over stdio, spoken to as an MCP client. +struct Agent { + run: Running, + next: u64, +} + +impl Agent { + fn start(home: &Home, book: &str) -> Self { + let mut c = home.cmd("mcp"); + c.args(["--book", book]); + let mut a = Self { + run: Running::start(c), + next: 1, + }; + a.request( + "initialize", + json!({"protocolVersion": "2025-06-18", "capabilities": {}, + "clientInfo": {"name": "test", "version": "0"}}), + 30, + ); + a.send(json!({"jsonrpc": "2.0", "method": "notifications/initialized"})); + a + } + + fn send(&mut self, message: Value) { + let stdin = self.run.child.stdin.as_mut().unwrap(); + writeln!(stdin, "{message}").unwrap(); + stdin.flush().unwrap(); + } + + fn request(&mut self, method: &str, params: Value, secs: u64) -> Value { + let id = self.next; + self.next += 1; + self.send(json!({"jsonrpc": "2.0", "id": id, "method": method, "params": params})); + let deadline = Instant::now() + Duration::from_secs(secs); + loop { + let left = deadline.saturating_duration_since(Instant::now()); + let line = self + .run + .lines + .recv_timeout(left) + .unwrap_or_else(|_| panic!("no answer to {method} in {secs} s")); + let v: Value = serde_json::from_str(&line).unwrap(); + if v.get("id") == Some(&json!(id)) { + return v; + } + } + } + + /// Call a tool: Ok(its structured result) or Err(the error's text). + fn call(&mut self, tool: &str, args: Value) -> Result { + let r = self.request("tools/call", json!({"name": tool, "arguments": args}), 60); + if let Some(e) = r.get("error") { + return Err(e["message"].as_str().unwrap_or_default().to_string()); + } + let result = &r["result"]; + if result["isError"] == true { + return Err(result["content"][0]["text"] + .as_str() + .unwrap_or_default() + .to_string()); + } + Ok(result["structuredContent"].clone()) + } +} + +/// Books `id` in the root as an older version wrote them (journal format +/// 1), with a chart, an entry and their vault key, before any server ran. +fn old_books(home: &Home, id: &str) -> PathBuf { + use ledger_core::{AccountType, Currency, Money, Side}; + use std::os::unix::fs::PermissionsExt; + let dir = home.book_dir(id); + std::fs::create_dir(&dir).unwrap(); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap(); + let mut s = ledger_store::Store::create_in_format( + &dir, + Currency::USD, + &ledger_store::JournalFormat::default(), + ) + .unwrap(); + let cash = s + .add_account("1000", "Cash", AccountType::Asset, None, false) + .unwrap(); + let cap = s + .add_account("3000", "Capital", AccountType::Equity, None, false) + .unwrap(); + let e = s + .create_entry(chrono_date(2026, 3, 1), "Capital in") + .unwrap(); + let m = Money::new(500.into(), Currency::USD); + s.add_line(e, cash, Side::Debit, m).unwrap(); + s.add_line(e, cap, Side::Credit, m).unwrap(); + s.post_entry(e).unwrap(); + drop(s); + let keys = dir.join("keys"); + std::fs::create_dir(&keys).unwrap(); + std::fs::set_permissions(&keys, std::fs::Permissions::from_mode(0o700)).unwrap(); + std::fs::write(keys.join("legacy.key"), "ab".repeat(32)).unwrap(); + std::fs::set_permissions( + keys.join("legacy.key"), + std::fs::Permissions::from_mode(0o600), + ) + .unwrap(); + dir +} + +fn chrono_date(y: i32, m: u32, d: u32) -> chrono::NaiveDate { + chrono::NaiveDate::from_ymd_opt(y, m, d).unwrap() +} + +fn journal(dir: &Path) -> Vec { + std::fs::read(dir.join("journal.jsonl")).unwrap() +} + +fn format_lines(dir: &Path) -> usize { + String::from_utf8(journal(dir)) + .unwrap() + .lines() + .filter(|l| l.contains(r#""type":"journal.format_set""#)) + .count() +} + +/// git, on the books' own backup repository: what it said. +fn archive_git(dir: &Path, args: &[&str]) -> String { + let out = Command::new("git") + .arg(format!("--git-dir={}", dir.join("archive.git").display())) + .args(args) + .output() + .unwrap(); + assert!(out.status.success(), "{out:?}"); + String::from_utf8(out.stdout).unwrap() +} + +fn a_setting() -> Value { + json!({"changes": [{"key": "company.name", "value": "Fernhill Labs LLC"}], + "decided_by": "Avery Quill"}) +} + +/// plan/served.md 3.9's `format_upgrade_waits_for_consent`, and D-39. +#[test] +fn format_upgrade_waits_for_consent() { + let home = Home::new("consent"); + let dir = old_books(&home, "acme"); + let before = journal(&dir); + // INVARIANT 3 (not at a terminal): a command that opens the books + // says they wait, and how to agree; it asks nothing. + let out = home + .cmd("token") + .args(["create", "--book", "acme"]) + .stdin(Stdio::null()) + .output() + .unwrap(); + assert!(out.status.success(), "{out:?}"); + let err = String::from_utf8_lossy(&out.stderr); + assert!( + err.contains("wait for a format upgrade") && err.contains("wanderbooks books upgrade acme"), + "{err}" + ); + let _held = serve(&home); + let out = home.books(&["list"]); + assert!( + String::from_utf8_lossy(&out.stdout).contains("acme open, waiting for a format upgrade"), + "{out:?}" + ); + let doctor = home.cmd("doctor").stdin(Stdio::null()).output().unwrap(); + assert!( + String::from_utf8_lossy(&doctor.stdout).contains("waiting for a format upgrade"), + "{doctor:?}" + ); + // And which database file the books use (one for each schema). + assert!( + String::from_utf8_lossy(&doctor.stdout) + .contains(&format!("database: {}", ledger_store::db_file())), + "{doctor:?}" + ); + + // INVARIANTS 1 and 2: the agent's view. + let mut a = Agent::start(&home, "acme"); + let start = a.call("books.start", json!({})).unwrap(); + assert_eq!(start["format_upgrade"]["guard"], "format_upgrade_pending"); + assert_eq!(start["format_upgrade"]["format"], 1); + assert_eq!(start["next_steps"][0]["tool"], "books.handbook"); + assert_eq!(start["next_steps"][0]["args"]["section"], "format-upgrade"); + let why = start["next_steps"][0]["why"].as_str().unwrap(); + assert!(why.starts_with("Tell the human first"), "{why}"); + let section = a + .call("books.handbook", json!({"section": "format-upgrade"})) + .unwrap(); + assert!( + section["section"]["text"] + .as_str() + .unwrap() + .contains("Only the owner can agree"), + "{section}" + ); + let refused = a.call("books.set_setting", a_setting()).unwrap_err(); + assert!(refused.contains("format_upgrade_pending"), "{refused}"); + assert!(refused.contains("no tool can"), "{refused}"); + assert_eq!(journal(&dir), before, "a refusal journals nothing"); + // What needs nothing new is made. + a.call( + "books.add_account", + json!({"code": "6000", "name": "Supplies", "account_type": "expense"}), + ) + .unwrap(); + let tools = a.request("tools/list", json!({}), 30); + let names: Vec<&str> = tools["result"]["tools"] + .as_array() + .unwrap() + .iter() + .map(|t| t["name"].as_str().unwrap()) + .collect(); + assert!(names.len() > 50, "{names:?}"); + for name in &names { + assert!( + !name.contains("upgrade") && !name.contains("consent"), + "a tool that could consent: {name}" + ); + } + assert_eq!(format_lines(&dir), 0, "no tool moved the format"); + + // INVARIANT 3: the go-ahead. Only at a terminal; a wrong passphrase, + // or none, changes nothing. + let out = home.books(&["upgrade", "acme"]); + assert_eq!(out.status.code(), Some(1), "{out:?}"); + assert!( + String::from_utf8_lossy(&out.stderr).contains("typed at a terminal"), + "{out:?}" + ); + let answer = home.lease_line(&format!( + "{}\n", + json!({"proto": 1, "intent": {"consent_format_upgrade": + {"book": "acme", "passphrase": "not the passphrase at all"}}}) + )); + assert!(answer.contains("isn't the owner passphrase"), "{answer}"); + let mut words = home.words("books"); + words.extend(["upgrade".into(), "acme".into()]); + let (code, shown) = home.at_terminal(&words, &[""]); + assert_eq!(code, 0, "{shown}"); + assert!(shown.contains("Not upgraded"), "{shown}"); + let wrong = "not the passphrase at all"; + let (code, shown) = home.at_terminal(&words, &[wrong]); + assert_eq!(code, 1, "{shown}"); + assert!( + shown.contains("error: not upgraded: that isn't the owner passphrase"), + "{shown}" + ); + assert!(!shown.contains(wrong), "the passphrase was shown: {shown}"); + assert!( + !dir.join("archive.git").exists(), + "no backup before consent" + ); + assert_eq!(format_lines(&dir), 0); + // Asked at every open until given (D-39): declined twice, asked twice. + let mut list = home.words("books"); + list.push("list".into()); + for _ in 0..2 { + let (code, shown) = home.at_terminal(&list, &["n"]); + assert_eq!(code, 0, "{shown}"); + assert!(shown.contains("Upgrade them now?"), "{shown}"); + assert!(shown.contains("Not now"), "{shown}"); + assert!(shown.contains("acme open, waiting"), "{shown}"); + } + // Never asked where the question can't be seen: at a terminal, but + // its output going to a file (or a pipe), the command only notes it on + // the terminal, asks nothing, and its output is its own. + let listed = home.dir.path().join("listed.txt"); + let mut to_file = list.clone(); + to_file.extend([">".to_string(), listed.display().to_string()]); + let (code, shown) = home.at_terminal(&to_file, &["n"]); + assert_eq!(code, 0, "{shown}"); + assert!(shown.contains("note: the books \"acme\" wait"), "{shown}"); + assert!(!shown.contains("Upgrade them now?"), "{shown}"); + assert_eq!( + std::fs::read_to_string(&listed).unwrap(), + "acme open, waiting for a format upgrade (`wanderbooks books upgrade acme`)\n" + ); + assert_eq!(format_lines(&dir), 0); + let before = journal(&dir); + + // INVARIANT 4: given, with the passphrase. + let (code, shown) = home.at_terminal(&words, &[PASS]); + assert_eq!(code, 0, "{shown}"); + assert!(!shown.contains(PASS), "the passphrase was shown: {shown}"); + assert!(shown.contains("Backed up first"), "{shown}"); + assert!( + shown.contains("Upgraded the books \"acme\" to this version's journal format"), + "{shown}" + ); + let after = journal(&dir); + assert!(after.starts_with(&before)); + assert_eq!(format_lines(&dir), 1, "the format moved once"); + let backup = archive_git(&dir, &["rev-parse", "HEAD"]); + assert_eq!( + archive_git(&dir, &["show", &format!("{}:journal.jsonl", backup.trim())]).as_bytes(), + before.as_slice(), + "the backup holds the books as they were" + ); + assert_eq!( + archive_git( + &dir, + &["show", &format!("{}:keys/legacy.key", backup.trim())] + ), + "ab".repeat(32), + "and their key" + ); + // The agent's session goes on: nothing waits, and the change is made. + let start = a.call("books.start", json!({})).unwrap(); + assert!(start.get("format_upgrade").is_none(), "{start}"); + assert_ne!(start["next_steps"][0]["args"]["section"], "format-upgrade"); + a.call("books.set_setting", a_setting()).unwrap(); + let verified = a.call("books.verify", json!({})).unwrap(); + assert_eq!(verified["database"], ledger_store::db_file(), "{verified}"); + assert!(verified.get("other_databases").is_none(), "{verified}"); + let out = home.books(&["list"]); + assert!( + String::from_utf8_lossy(&out.stdout).contains("acme open\n"), + "{out:?}" + ); + assert!(!String::from_utf8_lossy(&out.stderr).contains("format upgrade")); + // Asked again: nothing to do. + let (code, shown) = home.at_terminal(&words, &[]); + assert_eq!(code, 0, "{shown}"); + assert!( + shown.contains("already in this version's journal format"), + "{shown}" + ); + assert_eq!(format_lines(&dir), 1); +} diff --git a/crates/ledger-cli/tests/open_while_held.rs b/crates/ledger-cli/tests/open_while_held.rs index fc77d87..498378d 100644 --- a/crates/ledger-cli/tests/open_while_held.rs +++ b/crates/ledger-cli/tests/open_while_held.rs @@ -2724,6 +2724,7 @@ fn the_viewers_hold_new_books_and_close_the_books() { let common = Common { root: Some(home.root()), knobs: ledger_host::knobs::Knobs::parse(KNOBS).unwrap(), + ..Default::default() }; let hold = hold_for_viewer(&common, Some("acme".into())); assert!( diff --git a/crates/ledger-gui/Cargo.toml b/crates/ledger-gui/Cargo.toml index d80d8de..4144846 100644 --- a/crates/ledger-gui/Cargo.toml +++ b/crates/ledger-gui/Cargo.toml @@ -29,6 +29,7 @@ chrono = "0.4" rust_decimal = "1" serde = { version = "1", features = ["derive"] } serde_json = "1" +zeroize = "1" [dev-dependencies] chrono = "0.4" diff --git a/crates/ledger-gui/src/app.rs b/crates/ledger-gui/src/app.rs index 6177e38..16a6e73 100644 --- a/crates/ledger-gui/src/app.rs +++ b/crates/ledger-gui/src/app.rs @@ -1083,6 +1083,9 @@ impl eframe::App for App { }); return; }; + // Books waiting for a format upgrade: said above every page + // until it is done (owner.rs INVARIANT 5). + self.owner_menu.notice(ui, &books); let mut shell = Shell { receipts: &mut self.receipts, state: &mut self.state, diff --git a/crates/ledger-gui/src/books.rs b/crates/ledger-gui/src/books.rs index 82ccfe9..c19bb1c 100644 --- a/crates/ledger-gui/src/books.rs +++ b/crates/ledger-gui/src/books.rs @@ -324,6 +324,9 @@ pub struct Books { pub statements: Vec, /// Reconciliations, voided ones included, by date then id. pub reconciliations: Vec, + /// The format upgrade the books wait for, if they do (plan 3.9, D-39): + /// read from their journal, as the server reads it. + pub format_upgrade: Option, } /// A span of dates a party's money is summed over (the party page's and @@ -838,6 +841,7 @@ impl Books { links, statements, reconciliations, + format_upgrade: snap.journal_format().upgrade(), }; books.parties = books.party_infos(); books.kpis = books.compute_kpis(); diff --git a/crates/ledger-gui/src/owner.rs b/crates/ledger-gui/src/owner.rs index 5be774f..7b60f55 100644 --- a/crates/ledger-gui/src/owner.rs +++ b/crates/ledger-gui/src/owner.rs @@ -1,7 +1,8 @@ //! The viewer's Books menu (plan P2 "Viewer menu", D-19, D-32): the //! owner's own actions on the books, handed to the books server (back up -//! now, restore, new books, close the books) or to the owner's agent -//! (connect). The viewer itself still never writes the books. +//! now, restore, new books, close the books, the format upgrade) or to the +//! owner's agent (connect). The viewer itself still never writes the +//! books. //! //! ```text //! GOAL: the owner backs up, restores, starts new books, closes the books @@ -10,7 +11,9 @@ //! "New books…", "Close the books", "Connect Claude Code" and //! "Connect Codex"; "Nothing in the viewer writes a book"); D-32 //! (no confinement: the viewer asks the server itself, as the -//! command line does). +//! command line does); plan 3.9 and D-39 (the format upgrade: the +//! go-ahead asked at every open until given, with the owner +//! passphrase, and a standing notice). //! INVARIANTS: //! 1. Every action goes through the [`Owner`] the program gives the //! viewer (the books server, or the agent's own command): nothing @@ -20,6 +23,12 @@ //! would do), and done only for that backup, at that head. //! 4. Without an owner (the viewer on its own, older books dirs, the //! demo) the menu isn't there. +//! 5. Books waiting for a format upgrade show a standing notice above +//! every page, and the go-ahead is asked once each time the viewer +//! opens them: the owner passphrase, typed here and handed to the +//! owner's `upgrade_format` (the server checks it; nothing here +//! does). "Not now" changes nothing; a refusal (a wrong passphrase) +//! is said in the dialog, which stays. //! ``` use crate::books::Books; @@ -83,6 +92,9 @@ pub trait Owner: Send + Sync { fn restore(&self, plan: &RestorePlan) -> Result, String>; /// Let `agent` reach the books `book`. fn connect(&self, agent: Agent, book: &str) -> Result, String>; + /// The owner's go-ahead for the books `book`'s format upgrade, with the + /// owner passphrase they typed (the server checks it). + fn upgrade_format(&self, book: &str, passphrase: &str) -> Result, String>; } static OWNER: OnceLock> = OnceLock::new(); @@ -117,6 +129,48 @@ enum Dialog { Restore(Box), /// "Close the books": asked first. Close, + /// The format upgrade: the go-ahead, with the owner passphrase. + Upgrade(Box), +} + +/// The format upgrade's dialog (INVARIANT 5). +struct UpgradeForm { + /// What the books wait for. + upgrade: ledger_store::FormatUpgrade, + /// The owner passphrase, as typed (cleared once handed over). + passphrase: String, + /// The upgrade under way. + working: Option>, + /// Why the last try was refused. + problem: Option, +} + +impl UpgradeForm { + fn new(upgrade: ledger_store::FormatUpgrade) -> Self { + Self { + upgrade, + passphrase: String::new(), + working: None, + problem: None, + } + } +} + +impl Drop for UpgradeForm { + /// What was typed and not handed over is wiped from memory, as the + /// command line wipes what is typed at a terminal. + fn drop(&mut self) { + zeroize::Zeroize::zeroize(&mut self.passphrase); + } +} + +/// Wakes the window when dropped: a worker's last act, however it ends. +struct Wake(egui::Context); + +impl Drop for Wake { + fn drop(&mut self) { + self.0.request_repaint(); + } } /// A restore's check under way, and what it was asked: the backup, the @@ -147,21 +201,90 @@ pub struct OwnerMenu { dialog: Dialog, job: Option<(String, Receiver)>, done: Option<(String, Outcome)>, + /// The books whose format upgrade was asked for since the viewer + /// opened them (INVARIANT 5: once each open). + asked_upgrade: Option, } impl OwnerMenu { + /// The id of the books shown, if they are books of a data root: found + /// once for each load. + fn book_of(&mut self, books: &Arc) -> Option { + let key = Arc::as_ptr(books) as usize; + if self.book.as_ref().is_none_or(|(k, _)| *k != key) { + self.book = Some((key, book_id(books))); + } + self.book.as_ref().and_then(|(_, id)| id.clone()) + } + + /// The standing notice of a format upgrade the books wait for, above + /// the page, and the go-ahead asked for once each open (INVARIANT 5). + pub fn notice(&mut self, ui: &mut Ui, books: &Arc) { + if owner().is_none() { + return; + } + let Some(book) = self.book_of(books) else { + return; + }; + let Some(upgrade) = books.format_upgrade.clone() else { + return; + }; + if self.asked_upgrade.as_deref() != Some(book.as_str()) + && matches!(self.dialog, Dialog::None) + && self.job.is_none() + { + self.asked_upgrade = Some(book); + self.dialog = Dialog::Upgrade(Box::new(UpgradeForm::new(upgrade.clone()))); + } + let p = crate::theme::pal(ui.ctx()); + egui::Frame::new() + .fill(p.warning_soft) + .inner_margin(egui::Margin::symmetric(20, 10)) + .show(ui, |ui| { + ui.set_width(ui.available_width()); + ui.horizontal_wrapped(|ui| { + ui.label( + RichText::new(format!( + "{} These books wait for a format upgrade.", + icon::WARNING + )) + .font(crate::theme::semibold(13.0)) + .color(p.warning), + ); + ui.label( + RichText::new( + "An older version of wanderbooks wrote them. They work on their old format until you agree; what needs the new one is refused.", + ) + .font(crate::theme::regular(13.0)) + .color(p.text), + ); + let open = matches!(self.dialog, Dialog::None) && self.job.is_none(); + if ui + .add_enabled(open, egui::Button::new("Upgrade…")) + .on_hover_text("Give the go-ahead, with your owner passphrase: a verified backup is made first") + .clicked() + { + self.dialog = Dialog::Upgrade(Box::new(UpgradeForm::new(upgrade))); + } + }); + }); + } + /// The menu button and a note of the last outcome; its dialogs. pub fn ui(&mut self, ui: &mut Ui, books: &Arc, compact: bool) { let Some(owner) = owner() else { return; }; - let key = Arc::as_ptr(books) as usize; - if self.book.as_ref().is_none_or(|(k, _)| *k != key) { - self.book = Some((key, book_id(books))); - } - let Some(book) = self.book.as_ref().and_then(|(_, id)| id.clone()) else { + let Some(book) = self.book_of(books) else { return; }; + // Upgraded elsewhere meanwhile (at a terminal): nothing to ask. + if books.format_upgrade.is_none() + && let Dialog::Upgrade(form) = &self.dialog + && form.working.is_none() + { + self.dialog = Dialog::None; + } let p = crate::theme::pal(ui.ctx()); if let Some((what, rx)) = &self.job && let Ok(done) = rx.try_recv() @@ -228,6 +351,16 @@ impl OwnerMenu { )); } } + if let Some(upgrade) = &books.format_upgrade { + ui.separator(); + if ui + .button(format!("{} Upgrade the books' format…", icon::ARROW_FAT_UP)) + .on_hover_text("These books wait for a format upgrade: give the go-ahead, with your owner passphrase") + .clicked() + { + self.dialog = Dialog::Upgrade(Box::new(UpgradeForm::new(upgrade.clone()))); + } + } ui.separator(); if ui .button(format!("{} Close the books", icon::POWER)) @@ -242,7 +375,7 @@ impl OwnerMenu { if let Some((what, work)) = start { self.start(ui.ctx(), what, work); } - self.dialogs(ui.ctx(), &owner); + self.dialogs(ui.ctx(), &owner, &book); let said = if self.job.is_some() { Some(("Working…".to_string(), p.muted)) } else { @@ -278,12 +411,38 @@ impl OwnerMenu { self.done = None; } - /// The open dialog, if any. - fn dialogs(&mut self, ctx: &egui::Context, owner: &Arc) { + /// The open dialog, if any, for the books `book`. + fn dialogs(&mut self, ctx: &egui::Context, owner: &Arc, book: &str) { let mut close = false; let mut run: Option<(String, Box Outcome + Send>)> = None; match &mut self.dialog { Dialog::None => {} + Dialog::Upgrade(form) => { + let done = form.working.as_ref().and_then(|rx| match rx.try_recv() { + Ok(done) => Some(done), + Err(std::sync::mpsc::TryRecvError::Empty) => None, + // Its worker ended without a word (it died): the + // dialog, which holds the whole window, isn't left at + // work for good. + Err(std::sync::mpsc::TryRecvError::Disconnected) => Some(Err( + "it ended without an answer, so it may or may not have been done: if these books still wait, try again" + .into(), + )), + }); + if let Some(done) = done { + form.working = None; + match done { + Ok(lines) => { + self.done = Some(("Format upgrade".into(), Ok(lines))); + close = true; + } + Err(why) => form.problem = Some(why), + } + } + if !close { + upgrade_dialog(ctx, form, owner, book, &mut close); + } + } Dialog::Close => { egui::Modal::new(egui::Id::new("owner-close")).show(ctx, |ui| { ui.heading("Close the books?"); @@ -434,3 +593,74 @@ impl OwnerMenu { } } } + +/// The format upgrade's dialog (INVARIANT 5): what it is, the owner +/// passphrase, and the go-ahead or not now. +fn upgrade_dialog( + ctx: &egui::Context, + form: &mut UpgradeForm, + owner: &Arc, + book: &str, + close: &mut bool, +) { + let p = crate::theme::pal(ctx); + egui::Modal::new(egui::Id::new("owner-upgrade")).show(ctx, |ui| { + ui.set_max_width(520.0); + ui.heading("Upgrade these books' format?"); + ui.add_space(4.0); + ui.label(format!( + "An older version of wanderbooks wrote these books, in journal format {}. The upgrade adds {}.", + form.upgrade.from.format, + form.upgrade.adds().join(", ") + )); + ui.label("Until you agree they work as they are: everything that needs nothing new works, and what needs the new format is refused."); + ui.label("Upgrading first makes a backup of the books, their vault and keys included, and proves it whole; then it moves them to the new format, in one step. After that an older version of wanderbooks can't open them."); + ui.add_space(6.0); + let working = form.working.is_some(); + ui.label("Your owner passphrase:"); + let field = ui.add_enabled( + !working, + egui::TextEdit::singleline(&mut form.passphrase) + .password(true) + .desired_width(f32::INFINITY), + ); + if let Some(why) = &form.problem { + ui.colored_label(p.warning, why); + } + let go = field.lost_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter)); + ui.horizontal(|ui| { + let ready = !working && !form.passphrase.is_empty(); + if (ui + .add_enabled(ready, egui::Button::new("Upgrade")) + .clicked() + || (go && ready)) + && form.working.is_none() + { + // Handed over, and gone from the form (wiped once used). + let passphrase = zeroize::Zeroizing::new(std::mem::take(&mut form.passphrase)); + let (o, b) = (owner.clone(), book.to_string()); + let (tx, rx) = channel(); + let ctx = ui.ctx().clone(); + std::thread::spawn(move || { + // The window is woken however this ends (a panic + // too), once the answer, or its absence, can be seen. + let _wake = Wake(ctx); + let (tx, passphrase) = (tx, passphrase); + let _ = tx.send(o.upgrade_format(&b, &passphrase)); + }); + form.working = Some(rx); + form.problem = None; + } + if ui + .add_enabled(!working, egui::Button::new("Not now")) + .on_hover_text("They keep working as they are; you'll be asked again the next time they open") + .clicked() + { + *close = true; + } + if working { + ui.label("Backing up, then upgrading…"); + } + }); + }); +} diff --git a/crates/ledger-gui/tests/format_upgrade.rs b/crates/ledger-gui/tests/format_upgrade.rs new file mode 100644 index 0000000..34e2608 --- /dev/null +++ b/crates/ledger-gui/tests/format_upgrade.rs @@ -0,0 +1,396 @@ +//! The viewer and the format upgrade (owner.rs INVARIANT 5, plan 3.9, +//! D-39): books waiting for one show a standing notice above every page, +//! and the go-ahead is asked as the viewer opens them, with the owner +//! passphrase, handed to the owner the program gives the viewer (the +//! server checks it). "Not now" changes nothing; a refusal is said in the +//! dialog, which stays; books this version made, and older books dirs, +//! are never asked; the viewer writes nothing of the books. + +use chrono::NaiveDate; +use egui_kittest::Harness; +use egui_kittest::kittest::Queryable; +use ledger_core::Currency; +use ledger_gui::App; +use ledger_gui::books::Books; +use ledger_gui::data::{self, Source}; +use ledger_gui::nav::Route; +use ledger_gui::owner::{Agent, Owner, RestorePlan}; +use ledger_store::{JournalFormat, Store}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, OnceLock}; + +/// The owner passphrase the fake owner takes. +const PASS: &str = "correct horse battery"; +/// The passphrase on which the fake owner's worker dies, answering nothing. +const DIES: &str = "the worker dies on this one"; + +/// An owner that says what it was asked, and takes only PASS. +#[derive(Default)] +struct Fake { + asked: Mutex>, +} + +impl Owner for Fake { + fn close_books(&self) -> Result { + Err("not in these tests".into()) + } + fn new_books(&self, _id: &str, _currency: &str) -> Result { + Err("not in these tests".into()) + } + fn back_up_now(&self, _book: &str) -> Result, String> { + Err("not in these tests".into()) + } + fn plan_restore( + &self, + _source: &str, + _id: Option, + _replace: bool, + ) -> Result { + Err("not in these tests".into()) + } + fn restore(&self, _plan: &RestorePlan) -> Result, String> { + Err("not in these tests".into()) + } + fn connect(&self, _agent: Agent, _book: &str) -> Result, String> { + Err("not in these tests".into()) + } + fn upgrade_format(&self, book: &str, passphrase: &str) -> Result, String> { + // Slow enough to be seen at work. + std::thread::sleep(std::time::Duration::from_millis(300)); + // A worker that dies on the way: no answer comes back. + assert_ne!(passphrase, DIES, "the owner's worker died"); + self.asked + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(format!("upgrade {book} {passphrase}")); + if passphrase == PASS { + Ok(vec![ + "Backed up first, and proven whole with the vault and keys: the books as they were, through event 7 (backup 0123456789ab).".into(), + format!("Upgraded the books \"{book}\" to this version's journal format (journal format 2): now at event 8 (c4d3e2f1a0b9)."), + ]) + } else { + Err("that isn't the owner passphrase: nothing was changed".into()) + } + } +} + +/// One test at a time: they share the one owner and its record of calls. +fn alone() -> std::sync::MutexGuard<'static, ()> { + static ONE: Mutex<()> = Mutex::new(()); + ONE.lock().unwrap_or_else(|e| e.into_inner()) +} + +/// The one owner of this test binary (the viewer takes one, once). +fn fake() -> Arc { + static FAKE: OnceLock> = OnceLock::new(); + FAKE.get_or_init(|| { + let f = Arc::new(Fake::default()); + ledger_gui::owner::set_owner(f.clone()); + f + }) + .clone() +} + +fn asked(f: &Fake) -> Vec { + f.asked.lock().unwrap().clone() +} + +/// Books `acme`: in a data root (`root`) or an older books dir, in journal +/// format 1 (as an older version wrote them) or this version's (`old`). +fn books(root: bool, old: bool) -> (tempfile::TempDir, PathBuf, Books) { + let dir = tempfile::Builder::new() + .prefix("wb-gui-upgrade-") + .tempdir() + .unwrap(); + let book = if root { + std::fs::write(dir.path().join(ledger_store::root::MARKER_FILE), b"").unwrap(); + dir.path().join("books").join("acme") + } else { + dir.path().join("books-data") + }; + std::fs::create_dir_all(&book).unwrap(); + let format = if old { + JournalFormat::default() + } else { + ledger_store::format::current() + }; + drop(Store::create_in_format(&book, Currency::USD, &format).unwrap()); + let b = load(&book); + assert_eq!(b.format_upgrade.is_some(), old); + (dir, book, b) +} + +/// The books in `book`, as the viewer loads them. +fn load(book: &Path) -> Books { + data::load_once(&Source { + data_dir: book.to_path_buf(), + today: Some(NaiveDate::from_ymd_opt(2026, 10, 20).unwrap()), + }) + .unwrap() +} + +/// Every file under `dir`, with its bytes. +fn files(dir: &Path) -> Vec<(PathBuf, Vec)> { + let mut out = Vec::new(); + for e in std::fs::read_dir(dir).unwrap() { + let p = e.unwrap().path(); + if p.is_dir() { + out.extend(files(&p)); + } else { + out.push((p.clone(), std::fs::read(&p).unwrap())); + } + } + out.sort(); + out +} + +fn harness(b: Books) -> Harness<'static, App> { + let mut h = Harness::builder() + .with_size(egui::vec2(1200.0, 900.0)) + .build_eframe(|cc| App::with_books(&cc.egui_ctx, b, false, Route::Overview)); + h.run_steps(4); + h +} + +const ASKS: &str = "Upgrade these books' format?"; +const NOTICE: &str = "These books wait for a format upgrade."; + +/// Type into the dialog's passphrase field (the last text field). +fn type_passphrase(h: &mut Harness<'static, App>, text: &str) { + let field = h + .get_all_by_role(egui::accesskit::Role::PasswordInput) + .last() + .expect("a passphrase field"); + field.focus(); + h.run_steps(2); + h.get_all_by_role(egui::accesskit::Role::PasswordInput) + .last() + .unwrap() + .type_text(text); + h.run_steps(2); +} + +/// Run until the owner was asked `n` things, and the outcome shows. +fn settle(h: &mut Harness<'static, App>, f: &Fake, n: usize) { + for _ in 0..300 { + h.run_steps(2); + if asked(f).len() >= n { + h.run_steps(6); + return; + } + std::thread::sleep(std::time::Duration::from_millis(10)); + } + panic!("the owner wasn't asked: {:?}", asked(f)); +} + +/// D-39: asked as the viewer opens the books, and at every open until +/// given; "Not now" changes nothing and leaves the standing notice, whose +/// button asks again; nothing of the books is written. +#[test] +fn the_go_ahead_is_asked_at_every_open_with_a_standing_notice() { + let _alone = alone(); + let f = fake(); + let (_dir, book, b) = books(true, true); + let before = files(&book); + let n = asked(&f).len(); + let mut h = harness(b); + assert!(h.query_by_label(ASKS).is_some(), "asked as they open"); + assert!( + h.query_by_label_contains(NOTICE).is_some(), + "the standing notice" + ); + assert!(h.query_by_label("Upgrade").is_some()); + assert!( + h.query_by_label_contains("journal format 1. The upgrade adds journal format 2") + .is_some() + ); + h.get_by_label("Not now").click(); + h.run_steps(4); + assert!(h.query_by_label(ASKS).is_none(), "not now: gone"); + assert!( + h.query_by_label_contains(NOTICE).is_some(), + "the notice stays" + ); + h.run_steps(10); + assert!(h.query_by_label(ASKS).is_none(), "asked once each open"); + // The notice's button asks again. + h.get_by_label("Upgrade…").click(); + h.run_steps(4); + assert!(h.query_by_label(ASKS).is_some()); + h.get_by_label("Not now").click(); + h.run_steps(4); + // So does the Books menu. + h.get_by_label_contains(" Books").click(); + h.run_steps(3); + h.get_by_label_contains("Upgrade the books' format").click(); + h.run_steps(4); + assert!(h.query_by_label(ASKS).is_some()); + drop(h); + // Opened again: asked again. + let h = harness(load(&book)); + assert!(h.query_by_label(ASKS).is_some(), "asked at the next open"); + drop(h); + assert_eq!(asked(&f).len(), n, "declining asked the owner nothing"); + let after: Vec<_> = files(&book) + .into_iter() + .filter(|(p, _)| !p.starts_with(book.join("ops"))) + .collect(); + let before: Vec<_> = before + .into_iter() + .filter(|(p, _)| !p.starts_with(book.join("ops"))) + .collect(); + assert_eq!(after, before, "the viewer wrote nothing"); +} + +/// The go-ahead: nothing is sent without a passphrase; the one typed +/// goes to the owner as typed (which the server checks); a refusal is said +/// in the dialog, which stays, its field emptied; the right one closes it +/// and what was done is said. +#[test] +fn the_passphrase_goes_to_the_owner_and_a_refusal_is_said() { + let _alone = alone(); + let f = fake(); + let (_dir, _book, b) = books(true, true); + let mut h = harness(b); + let n = asked(&f).len(); + h.get_by_label("Upgrade").click(); + h.run_steps(4); + assert_eq!(asked(&f).len(), n, "nothing typed, nothing sent"); + type_passphrase(&mut h, "not the passphrase at all"); + h.get_by_label("Upgrade").click(); + h.run_steps(2); + assert!( + h.query_by_label_contains("Backing up, then upgrading") + .is_some() + ); + settle(&mut h, &f, n + 1); + assert_eq!(asked(&f)[n], "upgrade acme not the passphrase at all"); + assert!(h.query_by_label(ASKS).is_some(), "the dialog stays"); + assert!( + h.query_by_label("that isn't the owner passphrase: nothing was changed") + .is_some() + ); + type_passphrase(&mut h, PASS); + h.get_by_label("Upgrade").click(); + settle(&mut h, &f, n + 2); + assert_eq!(asked(&f)[n + 1], format!("upgrade acme {PASS}")); + assert!(h.query_by_label(ASKS).is_none(), "done: the dialog closes"); + assert!( + h.query_by_label_contains("Format upgrade: Backed up first") + .is_some() + ); +} + +/// An upgrade whose worker ends without an answer (it died on the way) is +/// said in the dialog, which can then be tried again or closed: the +/// dialog, which holds the whole window, is never left at work for good. +#[test] +fn an_upgrade_that_ends_without_an_answer_never_holds_the_window() { + let _alone = alone(); + let _ = fake(); + let (_dir, _book, b) = books(true, true); + let mut h = harness(b); + type_passphrase(&mut h, DIES); + h.get_by_label("Upgrade").click(); + h.run_steps(2); + assert!( + h.query_by_label_contains("Backing up, then upgrading") + .is_some() + ); + let said = "it ended without an answer"; + for _ in 0..300 { + h.run_steps(2); + if h.query_by_label_contains(said).is_some() { + break; + } + std::thread::sleep(std::time::Duration::from_millis(10)); + } + // The dialog grew by what it says: settled where it now stands before + // anything in it is clicked (as `settle` does). + h.run_steps(6); + assert!(h.query_by_label_contains(said).is_some(), "said"); + assert!( + h.query_by_label_contains("Backing up, then upgrading") + .is_none(), + "no longer at work" + ); + assert!(h.query_by_label(ASKS).is_some(), "the dialog stays"); + h.get_by_label("Not now").click(); + h.run_steps(4); + assert!(h.query_by_label(ASKS).is_none(), "and closes"); +} + +/// Books this version made wait for nothing, and older books dirs (no +/// owner's server for them) are never asked: no notice, no dialog, no +/// menu item. +#[test] +fn books_in_this_versions_format_and_older_books_dirs_are_never_asked() { + let _alone = alone(); + let _ = fake(); + for (root, old) in [(true, false), (false, true), (false, false)] { + let (_dir, _book, b) = books(root, old); + let mut h = harness(b); + assert!(h.query_by_label(ASKS).is_none(), "{root} {old}"); + assert!(h.query_by_label_contains(NOTICE).is_none(), "{root} {old}"); + if root { + h.get_by_label_contains(" Books").click(); + h.run_steps(3); + assert!( + h.query_by_label_contains("Upgrade the books' format") + .is_none() + ); + } + } +} + +/// Screenshots of the notice and the dialog, for eyes: set `SHOT_DIR`. +#[test] +fn screenshots() { + let Ok(out) = std::env::var("SHOT_DIR") else { + return; + }; + let _alone = alone(); + let out = PathBuf::from(out); + std::fs::create_dir_all(&out).unwrap(); + let f = fake(); + let (_dir, _book, b) = books(true, true); + let mut h = harness(b); + let shot = |h: &mut Harness<'static, App>, name: &str| { + h.run_steps(3); + h.render() + .unwrap() + .save(out.join(format!("{name}.png"))) + .unwrap(); + }; + shot(&mut h, "upgrade-ask"); + type_passphrase(&mut h, "not the passphrase at all"); + let n = asked(&f).len(); + h.get_by_label("Upgrade").click(); + h.run_steps(2); + shot(&mut h, "upgrade-working"); + settle(&mut h, &f, n + 1); + shot(&mut h, "upgrade-refused"); + type_passphrase(&mut h, DIES); + h.get_by_label("Upgrade").click(); + for _ in 0..300 { + h.run_steps(2); + if h.query_by_label_contains("ended without an answer") + .is_some() + { + break; + } + std::thread::sleep(std::time::Duration::from_millis(10)); + } + shot(&mut h, "upgrade-no-answer"); + h.get_by_label("Not now").click(); + shot(&mut h, "upgrade-notice"); + h.get_by_label_contains(" Books").click(); + shot(&mut h, "upgrade-menu"); + h.get_by_label_contains("Upgrade the books' format").click(); + h.run_steps(3); + type_passphrase(&mut h, PASS); + let n = asked(&f).len(); + h.get_by_label("Upgrade").click(); + settle(&mut h, &f, n + 1); + shot(&mut h, "upgrade-done"); +} diff --git a/crates/ledger-gui/tests/newer_journal.rs b/crates/ledger-gui/tests/newer_journal.rs index d8df45d..5b8f95d 100644 --- a/crates/ledger-gui/tests/newer_journal.rs +++ b/crates/ledger-gui/tests/newer_journal.rs @@ -40,7 +40,14 @@ fn books(tag: &str) -> DataDir { let dir = std::env::temp_dir().join(format!("ledger-gui-gate-{tag}-{}", std::process::id())); let _ = std::fs::remove_dir_all(&dir); let data = dir.join("books-data"); - let mut store = Store::create(&data, Currency::USD).unwrap(); + // In journal format 1, as before the format gate (this version's own + // books declare their format as they are made). + let mut store = Store::create_in_format( + &data, + Currency::USD, + &ledger_store::JournalFormat::default(), + ) + .unwrap(); store .add_account("1000", "Checking", AccountType::Asset, None, false) .unwrap(); diff --git a/crates/ledger-gui/tests/old_books.rs b/crates/ledger-gui/tests/old_books.rs index 1b08ecf..589688c 100644 --- a/crates/ledger-gui/tests/old_books.rs +++ b/crates/ledger-gui/tests/old_books.rs @@ -85,12 +85,15 @@ const SETTINGS: &str = "[company]\nname = \"Old Books Co\"\nfiscal_year_start = /// [`SETTINGS`] journaled into the books in `dir`, as the server adopts a /// settings file, unless they already keep their own (P1's books): the -/// viewer reads settings only from the journal (plan 3.6). +/// viewer reads settings only from the journal (plan 3.6). As adoption +/// does, the books move to this version's journal format first (an +/// adopted copy is new books of this version). fn write_settings(dir: &Path) { let mut store = ledger_store::Store::open(&dir.join("books-data")).unwrap(); if store.ledger().settings().is_journaled() { return; } + store.upgrade_format().unwrap(); let import = ledger_config::codec::import(SETTINGS, None, store.ledger().settings(), "USD").unwrap(); store diff --git a/crates/ledger-gui/tests/owner_menu.rs b/crates/ledger-gui/tests/owner_menu.rs index 50fc5be..48d0ca3 100644 --- a/crates/ledger-gui/tests/owner_menu.rs +++ b/crates/ledger-gui/tests/owner_menu.rs @@ -74,6 +74,10 @@ impl Owner for Fake { self.said(format!("connect {agent:?} {book}")); Ok(vec!["Claude Code reaches the books.".into()]) } + fn upgrade_format(&self, book: &str, _passphrase: &str) -> Result, String> { + self.said(format!("upgrade {book}")); + Err("not in these tests".into()) + } } /// One test at a time: they share the one owner and its record of calls. diff --git a/crates/ledger-gui/tests/parties.rs b/crates/ledger-gui/tests/parties.rs index 853065b..c09e512 100644 --- a/crates/ledger-gui/tests/parties.rs +++ b/crates/ledger-gui/tests/parties.rs @@ -649,15 +649,16 @@ fn journal_text_for_a_creditor() { )), "{texts:#?}" ); - // Declared with the feature already in force (the fixture's 1099 - // decisions'), sorted. - assert!( - texts - .iter() - .any(|(t, _)| t - == "Journal format 2 declared, with party_creditors, vendor_1099_decisions"), - "{texts:#?}" + // Declared when the books were created: this version's format, with + // every feature it knows (party_creditors among them), sorted. + let current = ledger_store::format::current(); + assert!(current.features.iter().any(|f| f == "party_creditors")); + let declared = format!( + "Journal format {} declared, with {}", + current.format, + current.features.join(", ") ); + assert!(texts.iter().any(|(t, _)| *t == declared), "{texts:#?}"); let mut h = Harness::builder() .with_size(egui::vec2(1440.0, 1400.0)) .build_eframe(|cc| App::with_books(&cc.egui_ctx, b, false, Route::Party(bank))); diff --git a/crates/ledger-host/src/commands.rs b/crates/ledger-host/src/commands.rs index 31d8f05..c42ef67 100644 --- a/crates/ledger-host/src/commands.rs +++ b/crates/ledger-host/src/commands.rs @@ -1,8 +1,9 @@ //! The owner's commands: `wanderbooks serve`, `close`, `books` (list, -//! create, adopt, restore, back-up), `token`, `connect`, `doctor` and -//! `passphrase` (plan 3.3, 3.7, 3.9, 5.5), and the same work for the +//! create, adopt, restore, back-up, upgrade), `token`, `connect`, `doctor` +//! and `passphrase` (plan 3.3, 3.7, 3.9, 5.5), and the same work for the //! viewer's Books menu (`new_books`, `back_up_now`, `plan_restore`, -//! `carry_out_restore`, `connect_agent`, `close_books`, `hold_for_viewer`). +//! `carry_out_restore`, `connect_agent`, `close_books`, `hold_for_viewer`, +//! `upgrade_format`). //! //! ```text //! GOAL: from a terminal, or the viewer's menu, the owner holds the books @@ -29,6 +30,14 @@ //! bridge reads its token from this keyring). //! 5. The viewer's hold lets go only when the owner closes the books; //! books that stop, or won't open, are tried again. +//! 6. Books waiting for a format upgrade (plan 3.9, D-39): the owner's +//! commands that open the books (`serve`, `books`, `token`, +//! `connect`) ask for the go-ahead once each time they run, at a +//! terminal that shows the question (input and output a terminal, in +//! the foreground: the question, then the owner passphrase, which only +//! the server checks); elsewhere they only say so, on stderr. Only a passphrase typed +//! at a terminal (or in the viewer) is sent, through the lease +//! socket's consent_format_upgrade; declining changes nothing. //! ``` use crate::knobs::{FLAG, Knobs}; @@ -45,6 +54,14 @@ const LIST_WAIT: Duration = Duration::from_secs(30); /// How long `close` waits to see the books closed. const CLOSE_WAIT: Duration = Duration::from_secs(120); +/// How long the owner's go-ahead for a format upgrade may take: the whole +/// books are backed up and proven first. +const UPGRADE_WAIT: Duration = Duration::from_secs(60 * 60); +/// How long a command waits to learn whether books wait for a format +/// upgrade before it gets on with its own work: a server slow to list +/// them (frozen, or busy) delays the command no more than this. +const OFFER_WAIT: Duration = Duration::from_secs(5); + /// Common flags: `--root DIR` and the test knobs. #[derive(Default, Clone)] pub struct Common { @@ -53,8 +70,16 @@ pub struct Common { pub root: Option, /// The test knobs. pub knobs: Knobs, + /// Ask about books waiting for a format upgrade when the books open + /// (INVARIANT 6): the owner's own commands; never the viewer's work + /// (its window asks) nor the agent's file commands. + pub ask_upgrades: bool, } +/// Whether this process asked about waiting format upgrades already: once +/// each time a command runs (INVARIANT 6). +static ASKED_UPGRADES: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + impl Common { /// Take a common flag (`Ok(true)`), or leave the argument. fn take(&mut self, arg: &str, args: &mut impl Iterator) -> Result { @@ -138,6 +163,12 @@ fn with_server_in(common: &Common, request: Request) -> Result<(Response, PathBu HoldError::Failed(why) => why, })?; let run_dir = opener.run_dir().to_path_buf(); + // The books are open: asked about a format upgrade first + // (INVARIANT 6). + if common.ask_upgrades && !ASKED_UPGRADES.swap(true, std::sync::atomic::Ordering::SeqCst) { + let run = run_dir.clone(); + let _ = tokio::task::spawn_blocking(move || offer_upgrades(&run)).await; + } let socket = run_dir.join(ledger_mcp::local::ADMIN_SOCKET); // A list is quick; a create or an adoption takes what it takes. let wait = matches!(request, Request::List | Request::TokenList).then_some(LIST_WAIT); @@ -270,6 +301,13 @@ pub fn serve( "The books are open. Press Ctrl-C to let go of them: they close when nothing else holds them." ); said_open = true; + // Asked about a format upgrade as they open (INVARIANT 6). + // A prompt is answered before Ctrl-C lets go: never left + // with the terminal's echo off. + if !ASKED_UPGRADES.swap(true, std::sync::atomic::Ordering::SeqCst) { + let run = opener.run_dir().to_path_buf(); + let _ = tokio::task::spawn_blocking(move || offer_upgrades(&run)).await; + } } else { println!("The books are open again."); } @@ -398,6 +436,7 @@ fn books_usage(prog: &str) { ); eprintln!(" {prog} restore BACKUP [--id ID] [--replace] [--yes]"); eprintln!(" {prog} back-up ID [--new-disk]"); + eprintln!(" {prog} upgrade ID"); eprintln!( " {prog} put-aside ID DATA-DIR [--vault DIR --vault-key FILE] [PATH...] [--yes]" ); @@ -410,6 +449,9 @@ fn books_usage(prog: &str) { eprintln!( " restore bring back a backup (a git repository's address, or BOOK.git in the backup folder), proven whole first: as new books, or replacing books (kept in legacy/)" ); + eprintln!( + " upgrade give the go-ahead, with the owner passphrase, for books an older wanderbooks wrote to move to this version's journal format (a verified backup with the vault and keys first)" + ); eprintln!( " put-aside move the old copies of adopted books (their data folder, their vault and its key, and other paths given: settings file, git history) whole into the legacy folder, once the books hold everything they do, byte for byte; nothing is deleted" ); @@ -464,6 +506,9 @@ pub fn books( } } let words: Vec<&str> = words.iter().map(String::as_str).collect(); + // The owner's command: asked about a format upgrade as the books open + // (INVARIANT 6), unless it is the upgrade itself. + common.ask_upgrades = !matches!(words.as_slice(), ["upgrade", ..]); let answer = match words.as_slice() { ["list"] => with_server(&common, Request::List), ["create", new] => with_server( @@ -494,6 +539,7 @@ pub fn books( ); } ["restore", source] => return restore(&common, source, id, replace, yes), + ["upgrade", book] => return upgrade(&common, book), ["put-aside", book, source, also @ ..] => { let also: Vec = also.iter().map(PathBuf::from).collect(); let old = ledger_store::root::OldCopies { @@ -531,10 +577,7 @@ pub fn books( } Ok(Response::Books(books)) => { for b in books { - match b.problem { - None => println!("{} open", b.id), - Some(why) => println!("{} didn't open: {why}", b.id), - } + println!("{}", book_line(&b)); } Ok(()) } @@ -562,6 +605,33 @@ pub fn books( } } +/// One book, as `books list` and `doctor` show it. +fn book_line(b: &admin::BookStatus) -> String { + match (&b.problem, &b.format_upgrade) { + (Some(why), _) => format!("{} didn't open: {why}", b.id), + (None, Some(_)) => format!( + "{} open, waiting for a format upgrade (`wanderbooks books upgrade {}`)", + b.id, b.id + ), + (None, None) => format!("{} open", b.id), + } +} + +/// The projection database a book uses, and the other schemas' files +/// beside it (left as they are for the versions that wrote them, never +/// deleted), as `doctor` says them; None when it didn't open. +fn database_line(b: &admin::BookStatus) -> Option { + let database = b.database.as_ref()?; + Some(if b.other_databases.is_empty() { + format!("database: {database}") + } else { + format!( + "database: {database} (also there, left for the versions that use them: {})", + b.other_databases.join(", ") + ) + }) +} + /// Back the books `book` up now (the server does it): what came of it, a /// line each; Err when it couldn't be done, or a target didn't take it. pub fn back_up_now(common: &Common, book: &str, new_disk: bool) -> Result, String> { @@ -585,6 +655,258 @@ pub fn back_up_now(common: &Common, book: &str, new_disk: bool) -> Result Vec { + vec![ + format!( + "The books \"{book}\" wait for a format upgrade: an older version of wanderbooks wrote them, in journal format {}. The upgrade adds {}.", + waiting.format, + waiting.adds.join(", ") + ), + "Until you agree they work on their old format: everything that needs nothing new works, and what needs the new format is refused.".into(), + "Upgrading first makes a backup of the books, their vault and keys included, and proves it whole; then it moves them to the new format, in one step. After that an older version of wanderbooks can't open them.".into(), + ] +} + +/// What a format upgrade did, a line each. +pub fn upgraded_lines(u: &admin::FormatUpgraded) -> Vec { + if u.added.is_empty() { + return vec![format!( + "The books \"{}\" are already in this version's journal format: nothing was changed.", + u.id + )]; + } + let mut lines = Vec::new(); + if let Some(b) = &u.backup { + lines.push(format!( + "Backed up first, and proven whole with the vault and keys: the books as they were, through event {} (backup {}).", + b.head.0, + short_hash(&b.commit) + )); + } + lines.push(format!( + "Upgraded the books \"{}\" to this version's journal format ({}): now at event {} ({}).", + u.id, + u.added.join(", "), + u.head.0, + short_hash(&u.head.1) + )); + lines +} + +/// Hand the owner's go-ahead for the books `book`'s format upgrade, with +/// the passphrase they typed, to the books' host (the lease socket's +/// consent_format_upgrade; INVARIANT 6), whose server checks it and does +/// it: what came of it, a line each. The books must be open. +fn consent_now(run: &Path, book: &str, passphrase: &str) -> Result, String> { + let asked = crate::opener::ask_once_within( + run, + Ask::Intent(Intent::ConsentFormatUpgrade { + book: book.to_string(), + passphrase: Some(admin::Passphrase(passphrase.to_string())), + }), + UPGRADE_WAIT, + ) + .map_err(|why| { + format!( + "{why}, so it may or may not have been done: `wanderbooks books list` says whether the books still wait for it" + ) + })?; + match asked { + Some(HostMsg::Done { detail }) => Ok(detail + .unwrap_or_default() + .lines() + .map(str::to_string) + .collect()), + // Refused: nothing was done (the server says so too). + Some(HostMsg::Refused(why)) => Err(format!("not upgraded: {why}")), + Some(HostMsg::Uncertain(why)) => Err(why), + // Sent, and the host gone without an answer (closed, or killed): + // its server may have done it all the same. + None => Err( + "the books' host gave no answer, so it may or may not have been done: `wanderbooks books list` says whether the books still wait for it" + .into(), + ), + Some(other) => Err(format!("the books' host answered oddly: {other:?}")), + } +} + +/// The owner's go-ahead for the books `book`'s format upgrade, with the +/// owner passphrase they typed (the viewer's dialog, `books upgrade`): the +/// books held open meanwhile. What came of it, a line each; or why not, +/// which says whether it may have been done all the same. +pub fn upgrade_format( + common: &Common, + book: &str, + passphrase: &str, +) -> Result, String> { + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .map_err(|e| e.to_string())?; + let (book, passphrase) = ( + book.to_string(), + zeroize::Zeroizing::new(passphrase.to_string()), + ); + let outcome = rt.block_on(async { + // Nothing is sent until the books are open. + let not_sent = |why: String| format!("not upgraded: {why}"); + let mut opener = + Opener::new(common.root.as_deref(), common.knobs, Kind::Cli, None).map_err(not_sent)?; + let lease = open(&mut opener).await.map_err(|e| match e { + HoldError::ClosedByOwner => { + "not upgraded: the books were closed just now (Close the books): nothing was done" + .to_string() + } + HoldError::Failed(why) => not_sent(why), + })?; + let run = opener.run_dir().to_path_buf(); + let asked = tokio::task::spawn_blocking(move || consent_now(&run, &book, &passphrase)) + .await + .map_err(|e| e.to_string())?; + drop(lease); + asked + }); + rt.shutdown_timeout(Duration::from_secs(1)); + outcome +} + +/// Ask the owner about every book waiting for a format upgrade (INVARIANT +/// 6), the books open: at a terminal, the go-ahead (then the owner +/// passphrase); elsewhere only a note. +fn offer_upgrades(run: &Path) { + let socket = run.join(ledger_mcp::local::ADMIN_SOCKET); + let Ok(Response::Books(books)) = admin::call_within(&socket, Request::List, Some(OFFER_WAIT)) + else { + return; + }; + for b in books { + if let Some(waiting) = &b.format_upgrade { + offer_upgrade(run, &b.id, waiting); + } + } +} + +/// [`offer_upgrades`] for the books `book`. The question comes unasked for, +/// into a command run for something else, so it is asked only where it is +/// seen and answered: at a terminal, the command's output going to it, and +/// run in the foreground (a command whose output goes to a file or a pipe, +/// or a job in the background, would wait unseen for an answer, and mix +/// the question into its output). +fn offer_upgrade(run: &Path, book: &str, waiting: &admin::FormatWaiting) { + // SAFETY: isatty, tcgetpgrp and getpgrp have no preconditions. + let seen = unsafe { libc::isatty(1) == 1 && libc::tcgetpgrp(0) == libc::getpgrp() }; + if !(seen && can_type_a_passphrase()) { + eprintln!( + "note: the books \"{book}\" wait for a format upgrade (to {}): until the owner agrees, with `wanderbooks books upgrade {book}` at a terminal or in the app, they work on their old format.", + waiting.adds.join(", ") + ); + return; + } + for line in upgrade_explained(book, waiting) { + println!("{line}"); + } + let not_now = || { + println!( + "Not now: the books keep working as they are, and you'll be asked again the next time they open (or run `wanderbooks books upgrade {book}`)." + ); + }; + if !matches!(confirmed("Upgrade them now?", false), Ok(true)) { + not_now(); + return; + } + let passphrase = match ask_owner_passphrase() { + Ok(Some(p)) => p, + Ok(None) => { + not_now(); + return; + } + Err(why) => { + eprintln!("error: {why}"); + return; + } + }; + match consent_now(run, book, &passphrase) { + Ok(lines) => { + for line in lines { + println!("{line}"); + } + } + Err(why) => eprintln!("error: {why}"), + } +} + +/// The owner passphrase, typed at the terminal for a format upgrade; None +/// when nothing was typed (no go-ahead). Never echoed or repeated. +fn ask_owner_passphrase() -> Result>, String> { + let typed = rpassword::prompt_password( + "The owner passphrase, to give the go-ahead (Enter alone: not now): ", + ) + .map(zeroize::Zeroizing::new) + .map_err(|e| format!("nothing was read ({})", e.kind()))?; + Ok((!typed.is_empty()).then_some(typed)) +} + +/// `wanderbooks books upgrade ID` (INVARIANT 6): the owner's go-ahead for +/// the books' format upgrade, at a terminal. +fn upgrade(common: &Common, book: &str) -> Result<(), Box> { + let fail = |why: String| -> Result<(), Box> { + eprintln!("error: {why}"); + Err(exit(1)) + }; + let book = match ledger_store::root::BookId::parse(book) { + Ok(id) => id.to_string(), + Err(e) => return fail(e.to_string()), + }; + let listed = match with_server(common, Request::List) { + Ok(Response::Books(books)) => books, + Ok(Response::Refused(why)) | Err(why) => return fail(why), + Ok(other) => return fail(format!("the books server answered oddly: {other:?}")), + }; + let Some(status) = listed.into_iter().find(|b| b.id == book) else { + return fail(format!( + "there are no books called \"{book}\": `wanderbooks books list` lists them" + )); + }; + if let Some(why) = status.problem { + return fail(format!("the books \"{book}\" didn't open: {why}")); + } + let Some(waiting) = status.format_upgrade else { + println!( + "The books \"{book}\" are already in this version's journal format: nothing to do." + ); + return Ok(()); + }; + for line in upgrade_explained(&book, &waiting) { + println!("{line}"); + } + if !can_type_a_passphrase() { + return fail(format!( + "the go-ahead is the owner's, with the owner passphrase typed at a terminal: run `wanderbooks books upgrade {book}` in one (or agree in the app)" + )); + } + let passphrase = match ask_owner_passphrase() { + Ok(Some(p)) => p, + Ok(None) => { + println!( + "Not upgraded: the books keep working as they are, and you'll be asked again." + ); + return Ok(()); + } + Err(why) => return fail(why), + }; + match upgrade_format(common, &book, &passphrase) { + Ok(lines) => { + for line in lines { + println!("{line}"); + } + Ok(()) + } + Err(why) => fail(why), + } +} + /// "New books…": new, empty books `id` in `currency` (the server makes /// them). pub fn new_books(common: &Common, id: &str, currency: &str) -> Result { @@ -745,6 +1067,8 @@ pub fn connect( eprintln!("usage: {prog} claude|codex --book ID"); return Err(exit(2)); }; + // The owner's command (INVARIANT 6). + common.ask_upgrades = true; match connect_agent(&common, agent, &book) { Ok(lines) => { for line in lines { @@ -1474,6 +1798,11 @@ fn adopted_lines(a: &admin::Adopted) -> Vec { if a.key_copied { lines.push(format!("TIN vault: {} records and its key.", a.vault_files)); } + if a.format_upgraded { + lines.push( + "The copy is in this version's journal format (the original keeps its own).".into(), + ); + } if a.settings_imported > 0 { lines.push(format!("{} settings moved in.", a.settings_imported)); } @@ -1573,9 +1902,9 @@ pub fn doctor( } if let Ok(admin::Response::Books(books)) = listed { for b in books { - match b.problem { - None => println!(" {} open", b.id), - Some(why) => println!(" {} didn't open: {why}", b.id), + println!(" {}", book_line(&b)); + if let Some(line) = database_line(&b) { + println!(" {line}"); } } // Agents' tokens: how many each book has. @@ -1691,6 +2020,8 @@ pub fn token( _ => words.push(arg), } } + // The owner's command (INVARIANT 6). + common.ask_upgrades = true; let fail = |why: String| -> Box { eprintln!("error: {why}"); exit(1) @@ -2456,6 +2787,7 @@ mod tests { settings_imported: 0, settings_ignored: false, source_lock_created: false, + format_upgraded: false, problem: None, } } @@ -2493,6 +2825,7 @@ mod tests { vault_files: 3, key_copied: true, settings_imported: 6, + format_upgraded: true, ..adopted() }); assert_eq!( @@ -2500,6 +2833,7 @@ mod tests { [ "Adopted as \"old\": 5 stored files copied, 4 of them checked against the books, 1 the books don't name (kept), 2 unfinished or hidden files left behind.", "TIN vault: 3 records and its key.", + "The copy is in this version's journal format (the original keeps its own).", "6 settings moved in.", "Head: event 4 (0123456789ab).", ] @@ -2512,6 +2846,101 @@ mod tests { assert!(empty.contains(&"TIN vault: 0 records and its key.".to_string())); } + /// What the owner is told of a format upgrade: what they wait for, + /// what was done (the backup first, then the format), and that books + /// already in this version's format had nothing done. + #[test] + fn a_format_upgrade_is_told_in_plain_words() { + let waiting = admin::FormatWaiting { + format: 1, + features: Vec::new(), + adds: vec!["journal format 2".into()], + }; + let told = upgrade_explained("acme", &waiting).join(" "); + for says in [ + "\"acme\" wait for a format upgrade", + "in journal format 1", + "The upgrade adds journal format 2.", + "Until you agree they work on their old format", + "vault and keys included", + "an older version of wanderbooks can't open them", + ] { + assert!(told.contains(says), "{says}: {told}"); + } + let status = + |problem: Option<&str>, waiting: Option| admin::BookStatus { + id: "acme".into(), + open: problem.is_none(), + problem: problem.map(str::to_string), + format_upgrade: waiting, + database: None, + other_databases: Vec::new(), + }; + assert_eq!(book_line(&status(None, None)), "acme open"); + assert_eq!( + book_line(&status(None, Some(waiting.clone()))), + "acme open, waiting for a format upgrade (`wanderbooks books upgrade acme`)" + ); + assert_eq!( + book_line(&status(Some("damaged"), Some(waiting))), + "acme didn't open: damaged" + ); + let done = admin::FormatUpgraded { + id: "acme".into(), + added: vec!["journal format 2".into()], + backup: Some(admin::UpgradeBackup { + commit: "0123456789abcdef".repeat(2), + head: (7, "a".repeat(64)), + }), + head: (8, "b".repeat(64)), + }; + assert_eq!( + upgraded_lines(&done), + [ + "Backed up first, and proven whole with the vault and keys: the books as they were, through event 7 (backup 0123456789ab).", + "Upgraded the books \"acme\" to this version's journal format (journal format 2): now at event 8 (bbbbbbbbbbbb).", + ] + ); + assert_eq!( + upgraded_lines(&admin::FormatUpgraded { + added: Vec::new(), + backup: None, + ..done + }), + [ + "The books \"acme\" are already in this version's journal format: nothing was changed." + ] + ); + } + + /// `doctor` names the database file a book uses, and any other + /// schema's beside it as left alone; nothing for books that didn't + /// open. + #[test] + fn doctor_names_the_database_in_use_and_the_others_left_alone() { + let mut b = admin::BookStatus { + id: "acme".into(), + open: true, + problem: None, + format_upgrade: None, + database: Some("wanderbooks.v13.db".into()), + other_databases: Vec::new(), + }; + assert_eq!( + database_line(&b).as_deref(), + Some("database: wanderbooks.v13.db") + ); + b.other_databases = vec!["wanderbooks.db".into(), "wanderbooks.v14.db".into()]; + assert_eq!( + database_line(&b).as_deref(), + Some( + "database: wanderbooks.v13.db (also there, left for the versions that use them: wanderbooks.db, wanderbooks.v14.db)" + ) + ); + b.database = None; + assert_eq!(database_line(&b), None); + } + #[test] fn the_log_lines_shown_name_no_folder_and_no_tin() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/ledger-host/src/host.rs b/crates/ledger-host/src/host.rs index a1b2ee5..132b29e 100644 --- a/crates/ledger-host/src/host.rs +++ b/crates/ledger-host/src/host.rs @@ -41,6 +41,12 @@ //! however long that takes. //! 11. The owner's close while restarting is refused (the restart is //! already under way; close again once it is done). +//! 12. The owner's go-ahead for a format upgrade (consent_format_upgrade, +//! its passphrase given) is handed to the open server's admin socket, +//! which checks the passphrase and does it (plan 3.9, D-39); the host +//! reads no books, and answers with what the server said, or that it +//! may or may not have been done. Refused while the books open or +//! close; the host keeps serving leases meanwhile. //! ``` use crate::knobs::{FLAG, Knobs}; @@ -550,15 +556,17 @@ impl Host { }, }); } - Event::Intent(intent, answer) => { - let (msg, closes) = self.intent(intent); - let _ = answer.send(msg); - if let Some(reason) = closes { - self.begin_close(reason); - idle_at = None; - if self.server_pid.is_none() { - return self.finish(connections, Ending::Closed).await; - } + Event::Intent(Intent::ConsentFormatUpgrade { book, passphrase }, answer) => { + self.consent(book, passphrase.unwrap_or_default(), answer); + } + Event::Intent(Intent::Close, answer) => { + let _ = answer.send(HostMsg::Done { + detail: Some("the books are closing".into()), + }); + self.begin_close(Reason::ClosedByOwner); + idle_at = None; + if self.server_pid.is_none() { + return self.finish(connections, Ending::Closed).await; } } Event::ServerReady => { @@ -682,20 +690,39 @@ impl Host { Ok(()) } - /// An owner intent: the answer, and whether it closes the books. - fn intent(&mut self, intent: Intent) -> (HostMsg, Option) { - match intent { - Intent::Close => ( - HostMsg::Done { - detail: Some("the books are closing".into()), - }, - Some(Reason::ClosedByOwner), - ), - Intent::ConsentFormatUpgrade { .. } => ( - HostMsg::Refused("no books are waiting for a format upgrade".into()), - None, - ), + /// The owner's go-ahead for a format upgrade (INVARIANT 12): handed to + /// the server's admin socket, off the loop (a backup comes first, and + /// may take a while); `answer` gets what came of it. + fn consent( + &self, + book: String, + passphrase: ledger_mcp::admin::Passphrase, + answer: oneshot::Sender, + ) { + if !matches!(self.phase, Phase::Open) { + let _ = answer.send(HostMsg::Refused( + "the books are closing: nothing was done".into(), + )); + return; } + if !self.ready { + let _ = answer.send(HostMsg::Refused( + "the books are still opening: nothing was done; try again once they are open" + .into(), + )); + return; + } + let socket = self.run.join(ledger_mcp::local::ADMIN_SOCKET); + tokio::spawn(async move { + let msg = tokio::task::spawn_blocking(move || consented(&socket, book, passphrase)) + .await + .unwrap_or_else(|_| { + HostMsg::Uncertain( + "the books' host stopped asking the books server, so it may or may not have been done: `wanderbooks books list` says whether the books still wait for it".into(), + ) + }); + let _ = answer.send(msg); + }); } /// Tell every lease why it ends, and end it (INVARIANT 5). @@ -737,6 +764,30 @@ impl Host { } } +/// Ask the server on `socket` to upgrade the books `book` with the +/// owner's passphrase (INVARIANT 12): what came of it, in the lease +/// socket's words. +fn consented(socket: &Path, book: String, passphrase: ledger_mcp::admin::Passphrase) -> HostMsg { + use ledger_mcp::admin::{CallError, Request, Response}; + match ledger_mcp::admin::call( + socket, + Request::UpgradeFormat { + id: book, + passphrase, + }, + ) { + Ok(Response::FormatUpgraded(u)) => HostMsg::Done { + detail: Some(crate::commands::upgraded_lines(&u).join("\n")), + }, + Ok(Response::Refused(why)) => HostMsg::Refused(why), + Ok(_) => HostMsg::Uncertain( + "the books server answered oddly, so it may or may not have been done: `wanderbooks books list` says whether the books still wait for it".into(), + ), + Err(CallError::NotSent(why)) => HostMsg::Refused(format!("{why}: nothing was done")), + Err(CallError::Unknown(why)) => HostMsg::Uncertain(why), + } +} + /// Take run/update.lock (exclusive), if it is free now. fn take_update_lock(run: &Path) -> Result { let file = std::fs::OpenOptions::new() diff --git a/crates/ledger-host/src/lease.rs b/crates/ledger-host/src/lease.rs index 10fa049..eadeb02 100644 --- a/crates/ledger-host/src/lease.rs +++ b/crates/ledger-host/src/lease.rs @@ -19,7 +19,9 @@ //! 2. Exactly two intents exist: close and consent_format_upgrade; any //! other request is refused. //! 3. consent_format_upgrade carries the owner passphrase; without one -//! it is refused before anything is done. +//! it is refused before anything is done. With one, the host hands it +//! to the books server, which checks it and upgrades (host.rs +//! INVARIANT 12; plan 3.9, D-39). //! 4. Before the host ends a lease it sends the reason; a lease that //! ends with no reason means the host is gone. //! 5. Each side accepts its own protocol version and the one before. @@ -114,9 +116,9 @@ pub enum Intent { ConsentFormatUpgrade { /// Which. book: String, - /// The owner passphrase (INVARIANT 3). + /// The owner passphrase (INVARIANT 3): Debug never shows it. #[serde(default)] - passphrase: Option, + passphrase: Option, }, } @@ -124,10 +126,11 @@ impl Intent { /// Refuse an intent that needs the passphrase and has none /// (INVARIANT 3). pub fn check_passphrase(&self) -> Result<(), String> { - let missing = |p: &Option| p.as_deref().is_none_or(|p| p.is_empty()); + let missing = + |p: &Option| p.as_ref().is_none_or(|p| p.0.is_empty()); match self { Intent::ConsentFormatUpgrade { passphrase, .. } if missing(passphrase) => Err( - "consenting to a format upgrade needs the owner passphrase, typed in the app" + "consenting to a format upgrade needs the owner passphrase, typed in the app or at a terminal" .into(), ), _ => Ok(()), @@ -296,7 +299,7 @@ mod tests { fn only_consent_needs_the_passphrase() { let consent = |p: Option<&str>| Intent::ConsentFormatUpgrade { book: "acme".into(), - passphrase: p.map(str::to_string), + passphrase: p.map(|p| ledger_mcp::admin::Passphrase(p.to_string())), }; assert!(consent(None).check_passphrase().is_err()); assert!(consent(Some("")).check_passphrase().is_err()); @@ -304,6 +307,23 @@ mod tests { assert!(Intent::Close.check_passphrase().is_ok()); } + /// INVARIANT 3: the passphrase a consent carries is sent as written, + /// and never shown by Debug, of the intent or the line that carried it + /// (an error or a log line that prints either names no passphrase). + #[test] + fn a_consent_never_shows_its_passphrase() { + let text = r#"{"proto":1,"intent":{"consent_format_upgrade":{"book":"acme","passphrase":"correct horse battery"}}}"#; + let hello = parse_hello(text).unwrap(); + let Ask::Intent(intent) = &hello.ask else { + panic!("{hello:?}") + }; + for shown in [format!("{hello:?}"), format!("{intent:?}")] { + assert!(!shown.contains("horse"), "{shown}"); + assert!(shown.contains("acme"), "{shown}"); + } + assert_eq!(line(&hello).trim_end(), text, "sent as written"); + } + #[test] fn versions_this_one_and_the_one_before() { assert!(accepts(PROTO)); diff --git a/crates/ledger-mcp/handbook/format-upgrade.md b/crates/ledger-mcp/handbook/format-upgrade.md new file mode 100644 index 0000000..e5594b3 --- /dev/null +++ b/crates/ledger-mcp/handbook/format-upgrade.md @@ -0,0 +1,7 @@ +# The format upgrade + +- A new version of wanderbooks can keep the books in a newer journal format than the version that wrote them. Books an older version wrote wait for a format upgrade: books.start says so (format_upgrade) and puts it first among its next steps, in every session until it is done. +- Until then they work on their old format: everything that needs nothing new works as before, and whatever would need the new format is refused with format_upgrade_pending, changing nothing. Never retry it, work around it, or post it another way. +- Only the owner can agree to the upgrade, with the owner passphrase: in the app, which asks each time it opens the books, or with `wanderbooks books upgrade ID` in a terminal. No tool can, and you never ask for, take or type the passphrase. Tell the human, in plain words: the books wait for their go-ahead, what was refused, and that saying no keeps the books working as they are. +- Before it upgrades, the server makes a backup of the books, their vault and keys included, and proves it whole; a backup that can't be made or proven upgrades nothing. Then the books move to the new format in one step, on the record. +- After the upgrade an older version of wanderbooks can't open these books; the backup made first keeps them as they were. Going back is the owner's choice, never yours. diff --git a/crates/ledger-mcp/src/admin.rs b/crates/ledger-mcp/src/admin.rs index bba6ad4..ef2ce85 100644 --- a/crates/ledger-mcp/src/admin.rs +++ b/crates/ledger-mcp/src/admin.rs @@ -1,6 +1,7 @@ //! The admin socket's messages (`run/admin.sock`, plan 3.1, 3.3, 3.9): how //! the command line asks the books server to list, create, adopt and -//! restore books, and to make, list and revoke agents' tokens. +//! restore books, to make, list and revoke agents' tokens, and how the +//! host hands it the owner's go-ahead for a format upgrade. //! //! ```text //! GOAL: the command line makes and adopts books without ever touching @@ -26,6 +27,9 @@ //! 6. A token's text is in exactly one answer, TokenCreated, to the //! request that made it; nothing else carries it (auth.rs //! INVARIANT 1). +//! 7. A format upgrade is asked with the owner passphrase, which the +//! server checks before anything is done (plan 3.9, D-39); no answer +//! carries it, and Debug never shows it. //! ``` use serde::{Deserialize, Serialize}; @@ -140,6 +144,16 @@ pub enum Request { }, /// The live tokens. TokenList, + /// The owner's go-ahead for the format upgrade of books waiting for + /// one (plan 3.9, D-39): a verified backup with the keys, then this + /// version's journal format declared. The host sends it, for the + /// lease socket's `consent_format_upgrade`. + UpgradeFormat { + /// The books. + id: String, + /// The owner passphrase, checked by the server (INVARIANT 7). + passphrase: Passphrase, + }, /// End a token. TokenRevoke { /// Its books. @@ -163,6 +177,73 @@ impl std::fmt::Debug for TokenText { } } +/// The owner passphrase in a request (and in the lease socket's consent, +/// which the host hands on as this): Debug never shows it (INVARIANT 7), +/// and it is wiped from memory when dropped, as the command line wipes +/// what is typed. +#[derive(Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(transparent)] +pub struct Passphrase(pub String); + +impl std::fmt::Debug for Passphrase { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("Passphrase(…)") + } +} + +impl Drop for Passphrase { + fn drop(&mut self) { + zeroize::Zeroize::zeroize(&mut self.0); + } +} + +/// A format upgrade books wait for (plan 3.9, D-39): written by an older +/// version, they open on their old format until the owner says go. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct FormatWaiting { + /// The journal format they are in. + pub format: u32, + /// The features they declared. + pub features: Vec, + /// What the upgrade adds, in words ("journal format 2"). + pub adds: Vec, +} + +impl From<&ledger_store::FormatUpgrade> for FormatWaiting { + fn from(u: &ledger_store::FormatUpgrade) -> Self { + Self { + format: u.from.format, + features: u.from.features.clone(), + adds: u.adds(), + } + } +} + +/// What a format upgrade did. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct FormatUpgraded { + /// The books. + pub id: String, + /// What it added, in words; empty when the books were already in this + /// version's format (nothing was done). + pub added: Vec, + /// The verified backup made first: the books as they were. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub backup: Option, + /// The books' head now. + pub head: (u64, String), +} + +/// The backup a format upgrade made first, proven to hold the books as +/// they were, their vault and keys too. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct UpgradeBackup { + /// Its commit in the books' own backup repository. + pub commit: String, + /// The head it holds. + pub head: (u64, String), +} + /// A token, as listed (never its text). #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct TokenInfo { @@ -200,6 +281,17 @@ pub struct BookStatus { /// Why it couldn't be opened, if it couldn't. #[serde(default, skip_serializing_if = "Option::is_none")] pub problem: Option, + /// The format upgrade it waits for, if it does. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub format_upgrade: Option, + /// The projection database's file in use (a name, no folder), if it + /// opened (ledger_store::db_file_for). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub database: Option, + /// Other schemas' projection files beside it, left as they are for the + /// versions that wrote them (never deleted). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub other_databases: Vec, } /// A setting an adoption would move in. @@ -259,6 +351,10 @@ pub struct Adopted { pub settings_ignored: bool, /// The source had no lock file, so one was made beside its journal. pub source_lock_created: bool, + /// The copy was moved to this version's journal format (the source's + /// is older, and keeps its own). + #[serde(default)] + pub format_upgraded: bool, /// Why the adopted books didn't open, if they didn't (they are kept; /// closing the books and opening them again tries again). #[serde(default, skip_serializing_if = "Option::is_none")] @@ -359,6 +455,8 @@ pub enum Response { Tokens(Vec), /// A token ended. TokenRevoked(TokenInfo), + /// A format upgrade made (or none needed). + FormatUpgraded(FormatUpgraded), /// Refused, with why (no path in the data root: INVARIANT 3). Refused(String), } @@ -409,6 +507,9 @@ pub fn call_within( Request::TokenCreate { .. } | Request::TokenRevoke { .. } => { Some("`wanderbooks token list` shows the tokens there are") } + Request::UpgradeFormat { .. } => { + Some("`wanderbooks books list` says whether the books still wait for it") + } }; let unknown = move |why: String| match look { None => CallError::Unknown(why), @@ -555,11 +656,37 @@ mod tests { #[test] fn answers_round_trip() { for r in [ - Response::Books(vec![BookStatus { - id: "acme".into(), - open: false, - problem: Some("damaged".into()), - }]), + Response::Books(vec![ + BookStatus { + id: "acme".into(), + open: false, + problem: Some("damaged".into()), + format_upgrade: None, + database: None, + other_databases: Vec::new(), + }, + BookStatus { + id: "beta".into(), + open: true, + problem: None, + format_upgrade: Some(FormatWaiting { + format: 1, + features: Vec::new(), + adds: vec!["journal format 2".into()], + }), + database: Some("wanderbooks.v13.db".into()), + other_databases: vec!["wanderbooks.db".into()], + }, + ]), + Response::FormatUpgraded(FormatUpgraded { + id: "beta".into(), + added: vec!["journal format 2".into()], + backup: Some(UpgradeBackup { + commit: "c".repeat(40), + head: (7, "ab".into()), + }), + head: (8, "cd".into()), + }), Response::Created { id: "acme".into(), problem: None, @@ -598,6 +725,42 @@ mod tests { } } + /// INVARIANT 7: the go-ahead for a format upgrade carries the owner + /// passphrase as written, and Debug never shows it; a book listed by + /// an earlier server (no format_upgrade) reads as waiting for none. + #[test] + fn an_upgrade_request_carries_the_passphrase_and_never_shows_it() { + let r: Envelope = serde_json::from_value(json!({"proto": 1, "op": "upgrade_format", + "id": "acme", "passphrase": "correct horse battery"})) + .unwrap(); + assert_eq!( + r.request, + Request::UpgradeFormat { + id: "acme".into(), + passphrase: Passphrase("correct horse battery".into()), + } + ); + let shown = format!("{r:?}"); + assert!(!shown.contains("horse"), "{shown}"); + assert!(shown.contains("Passphrase(…)"), "{shown}"); + assert!( + serde_json::from_value::( + json!({"proto": 1, "op": "upgrade_format", "id": "acme"}) + ) + .is_err(), + "an upgrade names the passphrase" + ); + let listed: BookStatus = + serde_json::from_value(json!({"id": "acme", "open": true})).unwrap(); + assert_eq!(listed.format_upgrade, None); + let up = ledger_store::FormatUpgrade { + from: ledger_store::JournalFormat::default(), + to: ledger_store::format::current(), + }; + let waiting = FormatWaiting::from(&up); + assert_eq!((waiting.format, waiting.adds), (1, up.adds())); + } + #[test] fn token_requests_read_as_written() { let r: Envelope = serde_json::from_value(json!({"proto": 1, "op": "token_create", diff --git a/crates/ledger-mcp/src/backup.rs b/crates/ledger-mcp/src/backup.rs index 3f2080e..899b71a 100644 --- a/crates/ledger-mcp/src/backup.rs +++ b/crates/ledger-mcp/src/backup.rs @@ -9,7 +9,9 @@ //! and the owner (and agent) can see that it did, or why not. //! SOURCE: plan 3.7 (triggers, "ops/backup.jsonl", books.backup, Needs //! attention for a target more than 24 h behind, drills), D-30 (a -//! git repository and a folder), D-31 (plain git). +//! git repository and a folder), D-31 (plain git), 3.9 and D-39 +//! (a verified local backup with the keys before a format +//! upgrade). //! INVARIANTS: //! 1. A backup is committed under the store's lock (ledger_store::backup //! INVARIANT 1; its stored files staged before, outside it) and @@ -41,6 +43,11 @@ //! theirs): it is paused for them, and said, until another is named. //! 8. A damaged backup record never keeps the books shut: it is set //! aside and a new one begun. +//! 9. A format upgrade (plan 3.9, D-39) is made only after the books are +//! committed to their archive and the archive proven to hold them +//! exactly as they are (journal, stored files, vault and keys), all +//! under the store's lock, so nothing is written in between; a backup +//! that can't be made or proven upgrades nothing. Both are records. //! ``` use crate::server::SharedStore; @@ -74,6 +81,76 @@ pub struct Backups { running: tokio::sync::Mutex<()>, } +/// What a format upgrade did ([`Backups::upgrade_format`]). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Upgraded { + /// The upgrade; None when the books were already in this version's + /// format (nothing was done). + pub upgrade: Option, + /// The verified backup made first (None when nothing was upgraded). + pub backup: Option, + /// The books' head now. + pub head: (u64, String), +} + +/// The backup a format upgrade made first: the books as they were. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UpgradeBackup { + /// Whether it was committed now (false: the last backup already held + /// the books as they were). + pub made: bool, + /// Its commit in the books' archive. + pub commit: String, + /// The head it holds: the books before the upgrade. + pub head: (u64, String), +} + +/// Why a format upgrade wasn't made: nothing was upgraded. +struct UpgradeFailed { + /// The backup committed on the way, if one was. + made: Option, + /// Why, in words. + why: String, +} + +impl UpgradeFailed { + /// The backup couldn't be made. + fn backup(e: BackupError) -> Self { + Self { + made: None, + why: format!( + "the backup the upgrade makes first couldn't be made ({e}): nothing was upgraded" + ), + } + } + + /// The backup `made` (now, or earlier) didn't prove whole. + fn proof(made: Option, e: BackupError) -> Self { + Self { + made, + why: format!( + "the backup the upgrade makes first didn't prove whole ({e}): nothing was upgraded" + ), + } + } + + /// Proven, and the books refused the upgrade. + fn store(made: Option, e: ledger_store::StoreError) -> Self { + Self { + made, + why: format!("{e}: nothing was upgraded"), + } + } + + /// The books server stopped, or the books are closing. + fn stopped(why: String) -> Self { + Self { + made: None, + why: format!("{why}: nothing was upgraded"), + } + } +} + /// One target's standing (INVARIANT 3). #[derive(Debug, Clone, Serialize, PartialEq, Eq)] pub struct TargetStatus { @@ -345,6 +422,129 @@ impl Backups { Ok(self.status().await) } + /// The format upgrade, after a verified local backup with the keys + /// (plan/served.md 3.9, D-39; INVARIANT 9). The caller has the owner's + /// go-ahead (the passphrase, checked by the server). + /// + /// Under the store's lock, so nothing is written between the backup + /// and the upgrade: the books committed to their archive (made + /// whatever targets the settings name, or none), the archive cloned + /// and proven to hold the books exactly as they are, journal, stored + /// files, vault and keys ([`backup::prove_archive`]); only then this + /// version's journal format declared (ledger_store + /// `Store::upgrade_format`). Books already in it: nothing is done. + /// Both the backup and the upgrade are records of `ops/backup.jsonl`; + /// the targets take them at the next backup, the books having moved. + pub async fn upgrade_format(&self) -> Result { + let _one = self.running.lock().await; + let (store, dir, book) = (self.store.clone(), self.dir.clone(), self.book.clone()); + let done = tokio::task::spawn_blocking(move || { + // Books that wait for nothing are left as they are: not even + // their archive is made or staged (asked again under the lock + // below; only an upgrade moves the format, one at a time). + match store.with_store(|s| (s.journal_format().upgrade().is_none(), s.head())) { + Ok((true, head)) => { + return Ok(Upgraded { + upgrade: None, + backup: None, + head, + }); + } + Ok((false, _)) => {} + Err(why) => return Err(UpgradeFailed::stopped(why)), + } + // Stored files staged ahead, outside the lock (as a backup's). + let git = Git::find(); + if let Ok(git) = &git { + let _ = backup::stage_blobs(git, &dir); + } + store + .with_store_mut(|s| -> Result { + let head = s.head(); + let Some(waiting) = s.journal_format().upgrade() else { + return Ok(Upgraded { + upgrade: None, + backup: None, + head, + }); + }; + let git = git.map_err(UpgradeFailed::backup)?; + let made = backup::commit(&git, &dir, &book, head.clone()) + .map_err(UpgradeFailed::backup)?; + let scratch = ledger_store::ops::ops_dir(&dir).join("upgrade-check"); + let _ = std::fs::remove_dir_all(&scratch); + let proven = backup::prove_archive(&git, &dir, &head, &scratch) + .and_then(|p| match &made { + // The commit proven is the one just made. + Some(m) if m.commit != p.commit => Err(BackupError::Damaged( + "the backup it proved isn't the one it made".into(), + )), + _ => Ok(p), + }) + .map_err(|e| UpgradeFailed::proof(made.clone(), e))?; + let commit = proven.commit; + let upgrade = match s.upgrade_format() { + Ok(upgrade) => upgrade, + // Journaled, and what follows the line failed (its + // database row, a full disk): the books are in this + // version's format all the same, and are never told + // otherwise (the database catches up as the books + // next open). + Err(e) if s.journal_format().upgrade().is_none() => { + eprintln!( + "wanderbooks server: the format upgrade was journaled, then: {}", + hide(&e.to_string(), &dir) + ); + Some(waiting) + } + Err(e) => return Err(UpgradeFailed::store(made.clone(), e)), + }; + Ok(Upgraded { + upgrade, + backup: Some(UpgradeBackup { + made: made.is_some(), + commit, + head, + }), + head: s.head(), + }) + }) + .map_err(UpgradeFailed::stopped) + .and_then(|r| r) + }) + .await + .unwrap_or_else(|_| Err(UpgradeFailed::stopped("the upgrade stopped".into()))); + match done { + Ok(u) => { + if let (Some(b), Some(up)) = (&u.backup, &u.upgrade) { + if b.made { + self.record( + json!({"kind": "commit", "why": "format_upgrade", "ok": true, + "commit": b.commit, "head": {"seq": b.head.0, "hash": b.head.1}}), + ); + } + self.record(json!({"kind": "format_upgrade", "ok": true, + "backup": {"commit": b.commit, "head": {"seq": b.head.0, "hash": b.head.1}}, + "from": {"format": up.from.format, "features": up.from.features}, + "to": {"format": up.to.format, "features": up.to.features}, + "head": {"seq": u.head.0, "hash": u.head.1}})); + } + Ok(u) + } + Err(failed) => { + if let Some(m) = &failed.made { + self.record( + json!({"kind": "commit", "why": "format_upgrade", "ok": true, + "commit": m.commit, "head": {"seq": m.head.0, "hash": m.head.1}}), + ); + } + let why = hide(&failed.why, &self.dir); + self.record(json!({"kind": "format_upgrade", "ok": false, "error": why})); + Err(why) + } + } + } + /// Whether a drill is due (INVARIANT 5). pub async fn drill_due(&self) -> bool { let records = self.records(); diff --git a/crates/ledger-mcp/src/handbook.rs b/crates/ledger-mcp/src/handbook.rs index 0d421c5..0594243 100644 --- a/crates/ledger-mcp/src/handbook.rs +++ b/crates/ledger-mcp/src/handbook.rs @@ -83,6 +83,10 @@ pub const SECTIONS: &[(&str, &str)] = &[ ), ("viewer", include_str!("../handbook/viewer.md")), ("backups", include_str!("../handbook/backups.md")), + ( + "format-upgrade", + include_str!("../handbook/format-upgrade.md"), + ), ( "rules-that-change", include_str!("../handbook/rules-that-change.md"), diff --git a/crates/ledger-mcp/src/lib.rs b/crates/ledger-mcp/src/lib.rs index b5248a6..642813e 100644 --- a/crates/ledger-mcp/src/lib.rs +++ b/crates/ledger-mcp/src/lib.rs @@ -5,8 +5,9 @@ //! //! Usage: `ledger-mcp [--data-dir PATH] [--config PATH] [--books-toml PATH]` //! -//! The data dir holds `journal.jsonl` (the source of truth), `wanderbooks.db` -//! (the disposable SQLite projection), `blobs/` (content-addressed receipts), +//! The data dir holds `journal.jsonl` (the source of truth), +//! `wanderbooks.v{N}.db` (the disposable SQLite projection, a file for each +//! schema: ledger_store::db_file_for), `blobs/` (content-addressed receipts), //! and optionally `config.json`. If no store exists, one is created with the //! configured base currency. Company settings (name, fiscal year start, //! receipt threshold) come from the books dir's `wanderbooks.toml`, by diff --git a/crates/ledger-mcp/src/local.rs b/crates/ledger-mcp/src/local.rs index b10a1e9..e53c0c1 100644 --- a/crates/ledger-mcp/src/local.rs +++ b/crates/ledger-mcp/src/local.rs @@ -136,6 +136,13 @@ enum Book { exports: ledger_store::exports::Exports, /// Its backups (backup.rs). backups: Arc, + /// The format upgrade it waits for, if any (plan 3.9, D-39): read + /// from its journal as it opened, and cleared by the upgrade (only + /// the upgrade moves an open book's format). + waiting: Arc>>, + /// Its projection database's file, and other schemas' beside it, + /// by name, as it opened (`wanderbooks doctor` says them). + database: (String, Vec), }, /// Not opened, with why (INVARIANT 3). Failed(String), @@ -289,6 +296,8 @@ impl Books { } let mut live = Live::own(); live.sync(store.ledger().settings(), store.base_currency().code()); + let waiting = Arc::new(Mutex::new(store.journal_format().upgrade())); + let database = (store.database_file(), store.other_databases()); let layout = Layout::Root(self.root.path().to_path_buf()); let access = Arc::new( AccessLog::open(store.data_dir()) @@ -347,6 +356,8 @@ impl Books { uploads, exports, backups, + waiting, + database, }) } @@ -386,6 +397,22 @@ impl Books { Book::Open { .. } => None, Book::Failed(why) => Some(why.clone()), }, + format_upgrade: match book { + Book::Open { waiting, .. } => waiting + .lock() + .unwrap_or_else(|e| e.into_inner()) + .as_ref() + .map(admin::FormatWaiting::from), + Book::Failed(_) => None, + }, + database: match book { + Book::Open { database, .. } => Some(database.0.clone()), + Book::Failed(_) => None, + }, + other_databases: match book { + Book::Open { database, .. } => database.1.clone(), + Book::Failed(_) => Vec::new(), + }, }) .collect(), ), @@ -490,6 +517,7 @@ impl Books { confirmed, expect_head, } => self.restore(id, source, replace, confirmed, expect_head), + Request::UpgradeFormat { id, passphrase } => self.upgrade_format(id, &passphrase), Request::BackUpNow { id, new_disk } => { let id = match BookId::parse(&id) { Ok(id) => id, @@ -655,6 +683,7 @@ impl Books { settings_imported: r.settings_imported, settings_ignored: r.settings_ignored, source_lock_created: r.source_lock_created, + format_upgraded: r.format_upgraded, problem, }) } @@ -666,6 +695,74 @@ impl Books { } impl Books { + /// The owner's go-ahead for a format upgrade (plan 3.9, D-39; admin + /// INVARIANT 7): the owner passphrase checked (Argon2id, as the human + /// pages check it) before anything is done; then a verified backup + /// with the keys and the upgrade, under the books' lock + /// (backup::Backups::upgrade_format). Books already in this version's + /// format: nothing is done, and the answer says so. + fn upgrade_format(&self, id: String, passphrase: &admin::Passphrase) -> Response { + let id = match BookId::parse(&id) { + Ok(id) => id, + Err(e) => return Response::Refused(self.hide(e.to_string())), + }; + let (backups, waiting) = match self.held().get(id.as_str()) { + Some(Book::Open { + backups, waiting, .. + }) => (backups.clone(), waiting.clone()), + Some(Book::Failed(why)) => { + return Response::Refused(format!("those books didn't open: {why}")); + } + None => { + return Response::Refused(format!( + "there are no books called {:?}: `wanderbooks books list` lists them", + id.as_str() + )); + } + }; + let auth = match self.root.auth_dir() { + Ok(dir) => dir, + Err(e) => return Response::Refused(self.hide(e.to_string())), + }; + match ledger_store::passphrase::verify(&auth, &passphrase.0) { + Ok(true) => {} + Ok(false) => { + return Response::Refused( + "that isn't the owner passphrase: nothing was changed".into(), + ); + } + Err(e) => { + return Response::Refused(format!( + "{}: nothing was changed", + self.hide(e.to_string()) + )); + } + } + let rt = tokio::runtime::Handle::current(); + match rt.block_on(backups.upgrade_format()) { + Ok(done) => { + // The books are in this version's format now (or were). + *waiting.lock().unwrap_or_else(|e| e.into_inner()) = None; + if let Some(up) = &done.upgrade { + eprintln!( + "wanderbooks server: the books {id} were upgraded to this version's journal format ({}), after a verified backup", + up.adds().join(", ") + ); + } + Response::FormatUpgraded(admin::FormatUpgraded { + id: id.to_string(), + added: done.upgrade.map(|u| u.adds()).unwrap_or_default(), + backup: done.backup.map(|b| admin::UpgradeBackup { + commit: b.commit, + head: b.head, + }), + head: done.head, + }) + } + Err(why) => Response::Refused(self.hide(why)), + } + } + /// Restore a backup (admin INVARIANT 4; ledger_store::root INVARIANT /// 7): a preview, or the books restored and opened. Books it replaces /// are let go first, and opened again if the restore fails. @@ -2976,4 +3073,318 @@ mod tests { ); assert_eq!(bound(&books).len(), MAX_SESSIONS); } + + /// The owner passphrase the upgrade tests set. + const PASS: &str = "correct horse battery"; + + /// Books `id` as an older version wrote them (journal format 1), with + /// a chart, an entry and a vault key, held by the server. + fn old_books(books: &Books, id: &str) -> PathBuf { + let id = BookId::parse(id).unwrap(); + let dir = books.root.book_dir(&id); + std::fs::create_dir(&dir).unwrap(); + std::fs::set_permissions(&dir, std::os::unix::fs::PermissionsExt::from_mode(0o700)) + .unwrap(); + let mut s = + Store::create_in_format(&dir, ledger_core::Currency::USD, &Default::default()).unwrap(); + let cash = s + .add_account("1000", "Cash", ledger_core::AccountType::Asset, None, false) + .unwrap(); + let cap = s + .add_account( + "3000", + "Capital", + ledger_core::AccountType::Equity, + None, + false, + ) + .unwrap(); + let e = s + .create_entry(chrono::NaiveDate::from_ymd_opt(2026, 3, 1).unwrap(), "In") + .unwrap(); + let m = + ledger_core::Money::new(rust_decimal::Decimal::from(500), ledger_core::Currency::USD); + s.add_line(e, cash, ledger_core::Side::Debit, m).unwrap(); + s.add_line(e, cap, ledger_core::Side::Credit, m).unwrap(); + s.post_entry(e).unwrap(); + drop(s); + let keys = dir.join(ledger_store::root::KEYS_DIR); + std::fs::create_dir(&keys).unwrap(); + std::fs::write( + keys.join(ledger_store::root::LEGACY_KEY_FILE), + "ab".repeat(32), + ) + .unwrap(); + books.hold(&id, books.open(&id)); + dir + } + + /// The books `id` as listed. + fn listed(books: &Books, id: &str) -> admin::BookStatus { + match books.admin(Request::List) { + Response::Books(list) => list.into_iter().find(|b| b.id == id).unwrap(), + other => panic!("{other:?}"), + } + } + + /// Ask for the upgrade of `id` with `pass`, off the runtime (as the + /// admin socket does). + async fn upgrade(books: &Arc, id: &str, pass: &str) -> Response { + let (books, id, pass) = (books.clone(), id.to_string(), pass.to_string()); + tokio::task::spawn_blocking(move || { + books.admin(Request::UpgradeFormat { + id, + passphrase: admin::Passphrase(pass), + }) + }) + .await + .unwrap() + } + + /// git, on the books' own archive: what it said. + fn archive_git(dir: &Path, args: &[&str]) -> String { + let out = std::process::Command::new("git") + .arg(format!("--git-dir={}", dir.join("archive.git").display())) + .args(args) + .output() + .unwrap(); + assert!(out.status.success(), "{out:?}"); + String::from_utf8(out.stdout).unwrap() + } + + /// plan/served.md 3.9's `format_upgrade_waits_for_consent`, the + /// server's part: books an older version wrote are served on their old + /// format and listed as waiting; the go-ahead is refused without the + /// owner passphrase (checked before anything is done: no backup, no + /// line), and the books stay served as they were; with it, a backup of + /// the books as they were, their keys in it, is committed and proven, + /// then the whole format is declared in one line; asked again, nothing. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn format_upgrade_waits_for_consent() { + let home = tempfile::tempdir().unwrap(); + let books = books_in(home.path()); + let dir = old_books(&books, "acme"); + let journal = || std::fs::read(dir.join(ledger_store::JOURNAL_FILE)).unwrap(); + let before = journal(); + let waiting = listed(&books, "acme").format_upgrade.expect("it waits"); + // The database file it uses: this version's schema's own. + assert_eq!( + listed(&books, "acme").database.as_deref(), + Some(ledger_store::db_file().as_str()) + ); + // Format 1 waits for this version's whole format: format 2 and + // every feature it knows. + let adds = ledger_store::JournalFormat::default() + .upgrade() + .unwrap() + .adds(); + assert_eq!(adds[0], "journal format 2"); + assert_eq!(waiting.format, 1); + assert_eq!(waiting.adds, adds); + // No passphrase set yet, then a wrong one, then none: refused, + // nothing done. + match upgrade(&books, "acme", PASS).await { + Response::Refused(why) => { + assert!(why.contains("no owner passphrase is set yet"), "{why}"); + assert!(why.contains("nothing was changed"), "{why}"); + } + other => panic!("{other:?}"), + } + let auth = books.root.auth_dir().unwrap(); + ledger_store::passphrase::set(&auth, PASS, None).unwrap(); + for wrong in ["not the passphrase at all", ""] { + match upgrade(&books, "acme", wrong).await { + Response::Refused(why) => { + assert_eq!(why, "that isn't the owner passphrase: nothing was changed") + } + other => panic!("{other:?}"), + } + } + assert_eq!(journal(), before); + assert!( + !dir.join("archive.git").exists(), + "no backup before consent" + ); + assert!(listed(&books, "acme").format_upgrade.is_some()); + assert_eq!(books.held()["acme"].problem(), None, "still served"); + // Books that don't exist, or didn't open. + assert!(matches!( + upgrade(&books, "nobody", PASS).await, + Response::Refused(_) + )); + // The go-ahead. + let done = match upgrade(&books, "acme", PASS).await { + Response::FormatUpgraded(u) => u, + other => panic!("{other:?}"), + }; + assert_eq!(done.id, "acme"); + assert_eq!(done.added, adds); + let backup = done.backup.clone().expect("backed up first"); + let after = journal(); + assert!(after.starts_with(&before)); + let added: Vec = String::from_utf8(after[before.len()..].to_vec()) + .unwrap() + .lines() + .map(|l| serde_json::from_str(l).unwrap()) + .collect(); + assert_eq!(added.len(), 1, "one line"); + assert_eq!(added[0]["type"], "journal.format_set"); + let c = ledger_store::format::current(); + assert_eq!( + added[0]["payload"], + serde_json::json!({"format": c.format, "features": c.features}) + ); + assert_eq!(done.head.0, backup.head.0 + 1); + // The backup: the books as they were, keys and all, the latest + // commit of their archive. + assert_eq!( + archive_git(&dir, &["rev-parse", "HEAD"]).trim(), + backup.commit + ); + assert_eq!( + archive_git(&dir, &["show", &format!("{}:journal.jsonl", backup.commit)]).as_bytes(), + before.as_slice() + ); + assert_eq!( + archive_git( + &dir, + &["show", &format!("{}:keys/legacy.key", backup.commit)] + ), + "ab".repeat(32) + ); + // Both on the backup record. + let record = + std::fs::read_to_string(ledger_store::ops::ops_dir(&dir).join(crate::backup::LOG_FILE)) + .unwrap(); + assert!(record.contains(r#""kind":"format_upgrade""#), "{record}"); + assert!(record.contains(&backup.commit), "{record}"); + assert!( + !ledger_store::ops::ops_dir(&dir) + .join("upgrade-check") + .exists() + ); + // Waiting for nothing now; asked again, nothing is done. + assert_eq!(listed(&books, "acme").format_upgrade, None); + match upgrade(&books, "acme", PASS).await { + Response::FormatUpgraded(u) => { + assert!(u.added.is_empty() && u.backup.is_none(), "{u:?}"); + } + other => panic!("{other:?}"), + } + assert_eq!(journal(), after); + // Books this version made wait for nothing, and asking for their + // upgrade changes nothing of them: not even a backup repository. + create(&books, "beta"); + assert_eq!(listed(&books, "beta").format_upgrade, None); + let beta = books.root.book_dir(&BookId::parse("beta").unwrap()); + match upgrade(&books, "beta", PASS).await { + Response::FormatUpgraded(u) => { + assert!(u.added.is_empty() && u.backup.is_none(), "{u:?}"); + } + other => panic!("{other:?}"), + } + assert!(!beta.join("archive.git").exists(), "nothing was changed"); + } + + /// The backup is trusted only once proven, and only the one just made: + /// an archive whose head names a branch other than the one a backup's + /// clone takes (a damaged archive) takes the new commit where no + /// clone finds it, so what is proven isn't what was made: nothing is + /// upgraded, and it is recorded. Once the archive is right, its latest + /// commit (which already holds the books) is proven and the upgrade + /// made. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn no_upgrade_unless_the_backup_just_made_is_proven() { + let home = tempfile::tempdir().unwrap(); + let books = books_in(home.path()); + let dir = old_books(&books, "acme"); + ledger_store::passphrase::set(&books.root.auth_dir().unwrap(), PASS, None).unwrap(); + let store = match &books.held()["acme"] { + Book::Open { store, .. } => store.clone(), + Book::Failed(why) => panic!("{why}"), + }; + let (d, s) = (dir.clone(), store.clone()); + let first = tokio::task::spawn_blocking(move || { + let git = ledger_store::backup::Git::find().unwrap(); + let head = s.with_store(|s| s.head()).unwrap(); + ledger_store::backup::commit(&git, &d, "acme", head).unwrap() + }) + .await + .unwrap() + .expect("a first backup"); + archive_git(&dir, &["symbolic-ref", "HEAD", "refs/heads/elsewhere"]); + // A second later: the commit made now, of the same files, isn't + // byte for byte the first (git dates commits to the second). + tokio::time::sleep(std::time::Duration::from_millis(1100)).await; + let before = std::fs::read(dir.join(ledger_store::JOURNAL_FILE)).unwrap(); + match upgrade(&books, "acme", PASS).await { + Response::Refused(why) => { + assert!(why.contains("didn't prove whole"), "{why}"); + assert!(why.contains("nothing was upgraded"), "{why}"); + } + other => panic!("{other:?}"), + } + assert_eq!( + std::fs::read(dir.join(ledger_store::JOURNAL_FILE)).unwrap(), + before + ); + assert!(listed(&books, "acme").format_upgrade.is_some()); + let record = || { + std::fs::read_to_string(ledger_store::ops::ops_dir(&dir).join(crate::backup::LOG_FILE)) + .unwrap() + }; + assert!( + record().contains(r#""why":"format_upgrade""#), + "the commit made: {}", + record() + ); + assert!(record().contains(r#""ok":false"#), "{}", record()); + // The archive put right: its latest commit holds the books. + archive_git(&dir, &["symbolic-ref", "HEAD", "refs/heads/main"]); + match upgrade(&books, "acme", PASS).await { + Response::FormatUpgraded(u) => { + let b = u.backup.clone().expect("proven"); + assert_eq!(b.commit, first.commit, "{u:?}"); + assert_eq!(b.head, first.head); + assert_eq!(u.head.0, first.head.0 + 1); + } + other => panic!("{other:?}"), + } + assert_eq!(listed(&books, "acme").format_upgrade, None); + } + + /// A backup that can't be made upgrades nothing: the books stay on + /// their old format, served and waiting, and the failure is recorded. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn no_upgrade_without_a_backup() { + let home = tempfile::tempdir().unwrap(); + let books = books_in(home.path()); + let dir = old_books(&books, "acme"); + ledger_store::passphrase::set(&books.root.auth_dir().unwrap(), PASS, None).unwrap(); + // A file no book keeps: the backup refuses it. + std::fs::write(dir.join(".gitattributes"), "* -text\n").unwrap(); + let before = std::fs::read(dir.join(ledger_store::JOURNAL_FILE)).unwrap(); + match upgrade(&books, "acme", PASS).await { + Response::Refused(why) => { + assert!(why.contains("couldn't be made"), "{why}"); + assert!(why.contains("nothing was upgraded"), "{why}"); + assert!(!why.contains(&home.path().display().to_string()), "{why}"); + } + other => panic!("{other:?}"), + } + assert_eq!( + std::fs::read(dir.join(ledger_store::JOURNAL_FILE)).unwrap(), + before + ); + assert!(listed(&books, "acme").format_upgrade.is_some()); + let record = + std::fs::read_to_string(ledger_store::ops::ops_dir(&dir).join(crate::backup::LOG_FILE)) + .unwrap(); + assert!( + record.contains(r#""kind":"format_upgrade","ok":false"#) + || (record.contains(r#""kind":"format_upgrade""#) + && record.contains(r#""ok":false"#)), + "{record}" + ); + } } diff --git a/crates/ledger-mcp/src/server.rs b/crates/ledger-mcp/src/server.rs index 5e32265..43f98d2 100644 --- a/crates/ledger-mcp/src/server.rs +++ b/crates/ledger-mcp/src/server.rs @@ -176,6 +176,41 @@ impl SharedStore { Ok(f(store)) } + /// Run `f` on the store under its lock, writing: the owner's format + /// upgrade, after its backup (backup.rs). Blocking. Refused once the + /// books are closing or the server stopped (INVARIANTS 2, 3), counted + /// as a request while it runs, and one that panics stops the server + /// (INVARIANT 1). + pub fn with_store_mut( + self: &Arc, + f: impl FnOnce(&mut Store) -> R, + ) -> Result { + if self.stopped() { + return Err(STOPPED.to_string()); + } + // Counted before the closing check, as a tool's request is. + let _running = Running::new(self); + if self.closing() { + return Err("the books are closing".into()); + } + let mut store = self.store.blocking_lock(); + if self.stopped() { + return Err(STOPPED.to_string()); + } + let store = store.as_mut().ok_or(STOPPED)?; + /// Stops the server if `f` panics while it holds the store. + struct StopOnPanic<'a>(&'a SharedStore); + impl Drop for StopOnPanic<'_> { + fn drop(&mut self) { + if std::thread::panicking() { + self.0.stop(); + } + } + } + let _stop = StopOnPanic(self); + Ok(f(store)) + } + /// Run `f` on the store if no one holds it now (books just opened): /// never waits, so never blocks a runtime's thread. pub fn try_with_store(&self, f: impl FnOnce(&Store) -> R) -> Option { @@ -3078,10 +3113,10 @@ impl BooksServer { )); } } - // A payment method exists only under the payment_instruments - // feature (books.add_instrument declared it), so tagging the draft - // with one below declares nothing after the draft's first line - // (B-27). + // A payment method exists only in books that declare the + // payment_instruments feature (books.add_instrument needs it), so + // tagging the draft with one below can't be refused for the format + // after the draft's first line (B-27). let id = store .create_entry(date, &p.description) .map_err(|e| e.to_string())?; @@ -4301,8 +4336,10 @@ impl BooksServer { format!("journal hash chain ({} events)", report.events), "fresh replay matches the live ledger (every statement, assets, classes, payroll)" .to_string(), - "SQLite database matches a fresh projection of the journal, every table, row for row" - .to_string(), + format!( + "SQLite database ({}) matches a fresh projection of the journal, every table, row for row", + report.database + ), ]; if let Some(n) = blobs { checked.push(format!("{n} stored files match their SHA-256")); @@ -4318,6 +4355,8 @@ impl BooksServer { head: report.head, healed: report.healed, blobs_verified: blobs, + database: report.database, + other_databases: report.other_databases, checked, })) } @@ -7169,10 +7208,10 @@ impl BooksServer { .set_party_creditor(id, *a, *on) .map_err(|e| e.to_string())?; } - // The creditor lines' journal feature, declared before this - // command's first line, never between its lines (rules.md B-27: a - // refused format upgrade must not leave the name changed and the - // creditors not). + // The creditor lines' journal feature, asked for before this + // command's first line, never between its lines (rules.md B-27: + // books waiting for their format upgrade must not be left with the + // name changed and the creditors not). if !creditor.is_empty() { store .ensure_feature(ledger_store::format::PARTY_CREDITORS) @@ -7473,9 +7512,9 @@ impl BooksServer { }; let on_file = facts.is_some(); // The vendor record's W-9 flag follows (below): its feature is - // declared before this command's first line, never after it - // (B-27: a refusal to upgrade the format must not leave the W-9 - // recorded and the flag not). + // asked for before this command's first line, never after it + // (B-27: books waiting for their format upgrade must not be left + // with the W-9 recorded and the flag not). let vendor = store.ledger().get_party(id).and_then(|p| p.vendor); if vendor .and_then(|v| store.ledger().get_vendor(v)) @@ -10486,6 +10525,13 @@ mod tests { }; out.push(("NewerJournal".into(), newer(true).to_string())); out.push(("NewerJournal unverified".into(), newer(false).to_string())); + out.push(( + "FormatUpgradePending".into(), + ledger_store::StoreError::FormatUpgradePending { + need: "journal format 2".into(), + } + .to_string(), + )); out.push(( "RuleNotKnown".into(), ledger_core::form1099::default_threshold(9999) @@ -11139,7 +11185,8 @@ mod tests { let server = BooksServer::new(store(&books), ReceiptPolicy::default()); rt.block_on(async { let seq: Body = |s| s.lock().map(|st| st.journal_seq()); - assert_eq!(run(&server, seq).await.unwrap(), Ok(1), "{how}"); + // New books: store.created, then their journal format. + assert_eq!(run(&server, seq).await.unwrap(), Ok(2), "{how}"); let died = run(&server, body).await.unwrap_err(); assert_eq!(died.message, STOPPED, "{how}"); // Every later request is refused, before it runs. diff --git a/crates/ledger-mcp/src/tools/kept.rs b/crates/ledger-mcp/src/tools/kept.rs index c1c1bde..11884bf 100644 --- a/crates/ledger-mcp/src/tools/kept.rs +++ b/crates/ledger-mcp/src/tools/kept.rs @@ -194,6 +194,19 @@ pub struct SectionOut { pub title: String, } +/// A format upgrade the books wait for. +#[derive(Debug, Serialize, JsonSchema)] +pub struct FormatUpgradeOut { + /// The guard: format_upgrade_pending (what a refused change says). + pub guard: String, + /// The journal format the books are in. + pub format: u32, + /// What the upgrade adds, in words. + pub adds: Vec, + /// Who can give the go-ahead, and how. + pub how: String, +} + /// Something to do next. #[derive(Debug, Serialize, JsonSchema)] pub struct NextStepOut { @@ -235,6 +248,12 @@ pub struct StartOut { pub truncated: Vec, /// The handbook's sections (books.handbook {section}). pub handbook_sections: Vec, + /// The format upgrade these books wait for, if they do (plan 3.9, + /// D-39): an older version wrote them, so they work on their old + /// journal format, and whatever needs the new one is refused, until + /// the owner gives the go-ahead. No tool can: tell the human. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub format_upgrade: Option, /// What to do next, in order. pub next_steps: Vec, /// How the books are kept from agents: "same_user (advisory)", hidden @@ -778,6 +797,17 @@ impl BooksServer { why: why.into(), }; let mut next_steps = Vec::new(); + // Books an older version wrote wait for the owner's go-ahead to + // upgrade their format: first, at every start until it is given + // (D-39). Only the owner can give it. + let upgrade = store.journal_format().upgrade(); + if upgrade.is_some() { + next_steps.push(step( + "books.handbook", + Some(serde_json::json!({"section": "format-upgrade"})), + "Tell the human first: these books wait for a format upgrade, which only the owner can agree to (in the app, which asks when it opens the books, or `wanderbooks books upgrade` at a terminal, with the owner passphrase; a verified backup is made first). Until then they work on their old format and refuse what needs the new one (format_upgrade_pending). No tool can upgrade them.", + )); + } // Books that read a settings file move it in before anything else // changes a setting (books.set_setting refuses until then). if !journaled { @@ -897,6 +927,12 @@ impl BooksServer { title: handbook::title(md).to_string(), }) .collect(), + format_upgrade: upgrade.map(|u| FormatUpgradeOut { + guard: ledger_store::error::FORMAT_UPGRADE_PENDING.to_string(), + format: u.from.format, + adds: u.adds(), + how: "only the owner, with the owner passphrase: in the app (it asks when it opens the books) or `wanderbooks books upgrade ID` at a terminal; a verified backup with the keys is made first. No tool can.".to_string(), + }), next_steps, isolation: "same_user (advisory)".to_string(), }; @@ -968,7 +1004,7 @@ impl BooksServer { #[tool( name = "books.start", title = "Start here", - description = "Call first in every session. The books in brief: company, today in its time zone, fiscal year, the journal's head, whether they need onboarding, where settings come from, the rules in force (receipts, parties, approvals), every setting with who set it, this company's conventions (how it keeps its books: posting templates, policies, close steps), open items, the handbook's sections, and next steps. Read-only; at most 8 KB.", + description = "Call first in every session. The books in brief: company, today in its time zone, fiscal year, the journal's head, whether they need onboarding, where settings come from, the rules in force (receipts, parties, approvals), every setting with who set it, this company's conventions (how it keeps its books: posting templates, policies, close steps), open items, the handbook's sections, a format upgrade the books wait for (only the owner can agree to it: tell them), and next steps. Read-only; at most 8 KB.", annotations(read_only_hint = true, open_world_hint = false) )] pub(super) async fn start( diff --git a/crates/ledger-mcp/src/types.rs b/crates/ledger-mcp/src/types.rs index 1acdc63..cfb41f6 100644 --- a/crates/ledger-mcp/src/types.rs +++ b/crates/ledger-mcp/src/types.rs @@ -677,6 +677,14 @@ pub struct VerifyOut { pub healed: Option, /// Stored files re-hashed, when checked. pub blobs_verified: Option, + /// The books' database file verified (a file for each schema, named for + /// it: "wanderbooks.v13.db", or the older "wanderbooks.db" when it holds + /// this version's schema). + pub database: String, + /// Other schemas' database files beside it: left as they are for the + /// versions that use them, never deleted. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub other_databases: Vec, /// What was checked, in words. pub checked: Vec, } @@ -4012,7 +4020,7 @@ pub struct ReconciliationsOut { /// Parameters for `books.handbook`. #[derive(Debug, Default, Deserialize, JsonSchema)] pub struct HandbookParams { - /// The section to read: contract, entries, drafts-and-reversals, evidence-and-receipts, attachments, files-in-out, reconciliation, fixed-assets, prepaid-and-recurring, revenue-and-payouts, parties-and-tins, 1099, cpa-handoff, settings-and-conventions, viewer, backups, rules-that-change, control-roles, tracing-figures, or "all" for the whole handbook in order. Omit for the index. + /// The section to read: contract, entries, drafts-and-reversals, evidence-and-receipts, attachments, files-in-out, reconciliation, fixed-assets, prepaid-and-recurring, revenue-and-payouts, parties-and-tins, 1099, cpa-handoff, settings-and-conventions, viewer, backups, format-upgrade, rules-that-change, control-roles, tracing-figures, or "all" for the whole handbook in order. Omit for the index. #[serde(default)] pub section: Option, } diff --git a/crates/ledger-mcp/tests/mcp_stdio.rs b/crates/ledger-mcp/tests/mcp_stdio.rs index ca589e2..e95b8dd 100644 --- a/crates/ledger-mcp/tests/mcp_stdio.rs +++ b/crates/ledger-mcp/tests/mcp_stdio.rs @@ -1945,6 +1945,30 @@ fn old_books_name_each_closes_rule_and_true_up_at_the_next_close() { preview["lines"], json!(["Debit 4000 0.02", "Credit 3001 0.02"]) ); + // Books an older version wrote: the close counted from its regime's + // start waits for the owner's go-ahead (D-39), which no tool gives. + let refused = mcp.call_err( + "books.close_period", + json!({"period_id": dec["period_id"], "confirmed": true}), + ); + assert!( + refused.starts_with("format_upgrade_pending:") && refused.contains("members_cumulative"), + "{refused}" + ); + // The owner's go-ahead, as the app gives it (ledger-gui owner.rs, after + // the passphrase and a verified backup): the books move to this + // version's format. + drop(mcp); + ledger_store::Store::open(&books) + .unwrap() + .upgrade_format() + .unwrap() + .expect("old books wait for the upgrade"); + let mut mcp = Mcp::spawn_with( + &books, + &[std::ffi::OsStr::new("--books-toml"), toml.as_os_str()], + ); + init(&mut mcp); let closed = mcp.call( "books.close_period", json!({"period_id": dec["period_id"], "confirmed": true}), @@ -3124,11 +3148,9 @@ fn rename_account_end_to_end_defect_15() { ); let after = mcp.call("books.verify", json!({"blobs": false})); assert_eq!(after["status"], "ok"); - assert_eq!( - after["events"], - json!(events + 2), - "the feature and the rename" - ); + // New books declare account_renamed (with the rest of this version's + // format) as they are created: the rename journals only itself. + assert_eq!(after["events"], json!(events + 1), "the rename"); let chart = mcp.call("books.chart", json!({})); let acct = chart["accounts"] .as_array() diff --git a/crates/ledger-mcp/tests/spec_1099.rs b/crates/ledger-mcp/tests/spec_1099.rs index 303a616..8aaca8c 100644 --- a/crates/ledger-mcp/tests/spec_1099.rs +++ b/crates/ledger-mcp/tests/spec_1099.rs @@ -356,7 +356,25 @@ const CHART: &[(&str, &str, &str)] = &[ /// Books with the chart; cash is 1000, financing 2100. fn books(name: &str) -> Mcp { + books_in(name, false) +} + +/// [`books`], as P2 left them when `p2`: journal format 2 with none of +/// P3's features, books an older version wrote, waiting for their format +/// upgrade (rules.md B-27). +fn books_in(name: &str, p2: bool) -> Mcp { let dir = test_dir(name); + if p2 { + ledger_store::Store::create_in_format( + &dir.join("books-data"), + ledger_core::Currency::USD, + &ledger_store::JournalFormat { + format: 2, + features: Vec::new(), + }, + ) + .unwrap(); + } let toml = books_toml(&dir, "Spec Co", "financing_accounts = [\"2100\"]\n"); let mut m = Mcp::start(&dir, &toml); m.accounts(CHART); @@ -365,6 +383,23 @@ fn books(name: &str) -> Mcp { } impl Mcp { + /// The owner's go-ahead, as the app gives it (ledger-gui owner.rs, + /// after the passphrase and a verified backup; no tool can): the + /// server stopped, the books moved to this version's format, and + /// served again. + fn upgraded(mut self) -> Mcp { + let dir = self.books_dir.take().expect("books_dir"); + drop(self); + ledger_store::Store::open(&dir.join("books-data")) + .unwrap() + .upgrade_format() + .unwrap() + .expect("P2's books wait for the upgrade"); + let mut m = Mcp::start(&dir, &dir.join("wanderbooks.toml")); + m.books_dir = Some(dir); + m + } + /// The same books, served again (a restart). fn restart(mut self) -> Mcp { let dir = self.books_dir.take().expect("books_dir"); @@ -553,7 +588,11 @@ fn set_vendor_1099_says_what_a_decision_needs() { assert!(why.contains(needle), "{args}: {why}"); } assert_eq!(m.count("vendor.1099_set"), 0); - assert_eq!(m.count("journal.format_set"), 0, "nothing declared either"); + assert_eq!( + m.count("journal.format_set"), + 1, + "nothing declared either: only the books' own format, from their creation" + ); } /// P3 brief 1.5 and review R8: tin_on_file omitted keeps the W-9 flag; @@ -646,30 +685,37 @@ fn a_members_method_off_the_financing_accounts_warns() { assert!(out["warnings"].as_array().unwrap().is_empty(), "{out}"); } -/// B-27 and the format-upgrade gate (D-39): books.set_w9 on books that -/// never used the vendor_1099_decisions feature journals the W-9 and then -/// the vendor's W-9 flag, a line of that feature, so the feature is -/// declared before the command's first line, never between its lines -/// (where a refused upgrade would leave the W-9 recorded and the flag -/// not). +/// B-27 and the format-upgrade gate (D-39): books.set_w9 journals the W-9 +/// and then the vendor's W-9 flag, a line of the vendor_1099_decisions +/// feature, so it asks for the feature before the command's first line, +/// never between its lines. In books waiting for their format upgrade +/// (P2's) the whole command is refused, journaling nothing (never the W-9 +/// recorded and the flag not); once the owner upgrades them, as in books +/// this version made, it journals the W-9, then the flag, and declares +/// nothing. #[test] -fn set_w9_declares_the_w9_flags_feature_before_its_first_line() { - let mut m = books("set-w9-feature"); +fn set_w9_asks_for_the_w9_flags_feature_before_its_first_line() { + let w9 = json!({"party": "Harbor Design LLC", "received_on": "2026-03-02", + "entity_type": "LLC (partnership)", "tin_last4": "4321", "kept_at": "CPA portal"}); + let mut m = books_in("set-w9-feature", true); m.add_party("Harbor Design LLC", &["contractor"]); m.ok("books.add_vendor", json!({"name": "Harbor Design LLC"})); - let before = m.journal_types().len(); - m.ok( - "books.set_w9", - json!({"party": "Harbor Design LLC", "received_on": "2026-03-02", - "entity_type": "LLC (partnership)", "tin_last4": "4321", "kept_at": "CPA portal"}), + let why = m.refuse("books.set_w9", w9.clone()); + assert!( + why.starts_with("format_upgrade_pending:") && why.contains("vendor_1099_decisions"), + "{why}" ); - let added = m.journal_types()[before..].to_vec(); - assert_eq!( - added, - ["journal.format_set", "party.w9_set", "vendor.w9_flag_set"], - "the declaration comes first" - ); - assert_eq!(decision(&mut m, "Harbor Design LLC")["tin_on_file"], true); + assert_eq!(m.count("party.w9_set"), 0, "nothing half done"); + let mut fresh = books("set-w9-feature-new"); + fresh.add_party("Harbor Design LLC", &["contractor"]); + fresh.ok("books.add_vendor", json!({"name": "Harbor Design LLC"})); + for mut m in [m.upgraded(), fresh] { + let before = m.journal_types().len(); + m.ok("books.set_w9", w9.clone()); + let added = m.journal_types()[before..].to_vec(); + assert_eq!(added, ["party.w9_set", "vendor.w9_flag_set"]); + assert_eq!(decision(&mut m, "Harbor Design LLC")["tin_on_file"], true); + } } /// Defect 12 end to end, restarting between steps (P3 brief 8.2): payment diff --git a/crates/ledger-mcp/tests/spec_mcp.rs b/crates/ledger-mcp/tests/spec_mcp.rs index e63c40b..0013609 100644 --- a/crates/ledger-mcp/tests/spec_mcp.rs +++ b/crates/ledger-mcp/tests/spec_mcp.rs @@ -340,7 +340,7 @@ impl Mcp { walk(&p, out); } else if name.ends_with("-wal") || name.ends_with("-shm") { continue; - } else if name == ledger_store::DB_FILE { + } else if ledger_store::is_database_file(&name) { out.insert(p.clone(), db_digest(&p)); } else { let bytes = std::fs::read(&p).unwrap_or_default(); diff --git a/crates/ledger-mcp/tests/spec_party_totals.rs b/crates/ledger-mcp/tests/spec_party_totals.rs index a51ff45..a1d80e8 100644 --- a/crates/ledger-mcp/tests/spec_party_totals.rs +++ b/crates/ledger-mcp/tests/spec_party_totals.rs @@ -361,7 +361,25 @@ const CHART: &[(&str, &str, &str)] = &[ /// financing 2100; the chart; both members and the card's issuer in the /// directory. fn books(name: &str) -> Mcp { + books_in(name, false) +} + +/// [`books`], as P2 left them when `p2`: journal format 2 with none of +/// P3's features, books an older version wrote, waiting for their format +/// upgrade (rules.md B-27). +fn books_in(name: &str, p2: bool) -> Mcp { let dir = test_dir(name); + if p2 { + ledger_store::Store::create_in_format( + &dir.join("books-data"), + ledger_core::Currency::USD, + &ledger_store::JournalFormat { + format: 2, + features: Vec::new(), + }, + ) + .unwrap(); + } let toml = books_toml( &dir, "Fernhill Labs LLC", @@ -378,6 +396,25 @@ fn books(name: &str) -> Mcp { m } +impl Mcp { + /// The owner's go-ahead, as the app gives it (ledger-gui owner.rs, + /// after the passphrase and a verified backup; no tool can): the + /// server stopped, the books moved to this version's format, and + /// served again. + fn upgraded(mut self) -> Mcp { + let dir = self.books_dir.take().expect("books_dir"); + drop(self); + ledger_store::Store::open(&dir.join("books-data")) + .unwrap() + .upgrade_format() + .unwrap() + .expect("P2's books wait for the upgrade"); + let mut m = Mcp::start(&dir, &dir.join("wanderbooks.toml")); + m.books_dir = Some(dir); + m + } +} + /// A party's part of an entry as books.get_entry gives it: (resolved, /// direction, cash, in_kind). fn part(m: &mut Mcp, entry: u64, party: &str) -> (Value, Value, Value, Value) { @@ -560,7 +597,8 @@ fn defect14_party_totals_take_a_basis() { /// Defect 24 over the MCP: `books.update_party` `creditor_of` names the /// liabilities the company owes a party itself on (journaled as -/// party.creditor_set, the journal's feature declared first), replaces +/// party.creditor_set, under the journal feature the books declared when +/// they were created, so nothing is declared on the way), replaces /// the list, refuses what isn't a liability with nothing journaled; and a /// fee on the issuer's own card then reads neither, so the issuer is paid /// what the card payment paid, once. @@ -595,8 +633,8 @@ fn defect24_creditor_of_round_trips_and_a_card_fee_is_not_a_payment() { assert_eq!(m.count("party.creditor_set"), 1); assert_eq!( m.count("journal.format_set"), - before + 1, - "the feature declared once" + before, + "nothing declared on the way" ); // The same list again changes nothing. let journal = m.journal(); @@ -633,31 +671,40 @@ fn defect24_creditor_of_round_trips_and_a_card_fee_is_not_a_payment() { ); assert_eq!(p["creditor_of"], json!([])); assert_eq!(m.count("party.creditor_set"), 4); - assert_eq!(m.count("journal.format_set"), before + 1); + assert_eq!(m.count("journal.format_set"), before); let p = m.read_only("books.party", json!({"party": "Tobin Harlow"})); assert_eq!(p["party"]["creditor_of"], json!(["2100"])); assert_eq!(m.ok("books.verify", json!({}))["status"], "ok"); } -/// rules.md B-27: `books.update_party` declares the `party_creditors` -/// feature before the first line it writes, never between them: a name -/// change with the first creditor is journaled as the declaration, then -/// the name, then the creditor (a refused format upgrade then leaves -/// nothing half done). +/// rules.md B-27: `books.update_party` asks for the `party_creditors` +/// feature before the first line it writes, never between them. In books +/// waiting for their format upgrade (P2's), a name change with the first +/// creditor is refused whole, journaling nothing (never the notes changed +/// and the creditor not); once the owner upgrades them, as in books this +/// version made, it journals the change, then the creditor, and declares +/// nothing. #[test] -fn update_party_declares_the_creditor_feature_before_its_first_line() { - let mut m = books("feature-first"); - let start = m.journal_types().len(); - let p = m.ok( - "books.update_party", - json!({"party": "Harbor Bank", "notes": "the card's issuer", "creditor_of": ["2050"]}), +fn update_party_asks_for_the_creditor_feature_before_its_first_line() { + let change = + json!({"party": "Harbor Bank", "notes": "the card's issuer", "creditor_of": ["2050"]}); + let mut m = books_in("feature-first", true); + let why = m.refuse("books.update_party", change.clone()); + assert!( + why.starts_with("format_upgrade_pending:") && why.contains("party_creditors"), + "{why}" ); - assert_eq!(p["creditor_of"], json!(["2050"])); - assert_eq!( - m.journal_types()[start..].to_vec(), - ["journal.format_set", "party.updated", "party.creditor_set"], - ); - assert_eq!(m.ok("books.verify", json!({}))["status"], "ok"); + assert_eq!(m.count("party.updated"), 0, "nothing half done"); + for mut m in [m.upgraded(), books("feature-first-new")] { + let start = m.journal_types().len(); + let p = m.ok("books.update_party", change.clone()); + assert_eq!(p["creditor_of"], json!(["2050"])); + assert_eq!( + m.journal_types()[start..].to_vec(), + ["party.updated", "party.creditor_set"], + ); + assert_eq!(m.ok("books.verify", json!({}))["status"], "ok"); + } } /// What `books.post_entry` previews is what it posts: each party's diff --git a/crates/ledger-mcp/tests/spec_vault.rs b/crates/ledger-mcp/tests/spec_vault.rs index 71b1b01..f4616de 100644 --- a/crates/ledger-mcp/tests/spec_vault.rs +++ b/crates/ledger-mcp/tests/spec_vault.rs @@ -1378,8 +1378,8 @@ fn no_full_tin_leaves_the_vault_except_the_worksheet_when_asked() { ); seen.push(f.file_name().unwrap().to_string_lossy().into_owned()); } - for must in ["journal.jsonl", "wanderbooks.db"] { - assert!(seen.iter().any(|s| s == must), "{must} scanned: {seen:?}"); + for must in ["journal.jsonl".to_string(), ledger_store::db_file()] { + assert!(seen.contains(&must), "{must} scanned: {seen:?}"); } // The vault holds them encrypted only. let vault_files = files_under(&m.vault()); diff --git a/crates/ledger-store/src/backup.rs b/crates/ledger-store/src/backup.rs index a945f0a..c2d9b5b 100644 --- a/crates/ledger-store/src/backup.rs +++ b/crates/ledger-store/src/backup.rs @@ -966,6 +966,90 @@ pub fn drill( checked } +/// Prove the book's own archive holds the book in `book_dir` exactly as it +/// is now, its journal's head `head` (INVARIANT 3, and more): the archive +/// cloned into `scratch` (a new folder, removed after) and proven whole as +/// a restore proves a backup, its head `head`, its journal the book's byte +/// for byte, and its TIN vault and keys the book's, file for file and byte +/// for byte. What the format upgrade asks of the backup it makes first +/// (plan/served.md 3.9: "a verified local backup that includes the +/// keys"). The caller holds the store's lock, so the book can't move +/// while it is compared. +pub fn prove_archive( + git: &Git, + book_dir: &Path, + head: &(u64, String), + scratch: &Path, +) -> Result { + let archive = book_dir.join(ARCHIVE_DIR); + let source = archive.display().to_string(); + let proven = verify(git, &source, scratch).and_then(|v| { + if v.head != *head { + return Err(BackupError::Damaged(format!( + "it holds the books through event {}, and they are at event {}", + v.head.0, head.0 + ))); + } + if std::fs::read(book_dir.join(crate::JOURNAL_FILE))? + != std::fs::read(v.tree.join(crate::JOURNAL_FILE))? + { + return Err(BackupError::Damaged( + "its journal isn't the books' own, byte for byte".into(), + )); + } + for kept in [crate::root::VAULT_DIR, crate::root::KEYS_DIR] { + if held_files(book_dir, kept)? != held_files(&v.tree, kept)? { + return Err(BackupError::Damaged(format!( + "its {} isn't the books' own, file for file", + if kept == crate::root::KEYS_DIR { + "keys folder" + } else { + "TIN vault" + } + ))); + } + } + let git_dir = format!("--git-dir={}", v.tree.join(".git").display()); + let commit = git.run("rev-parse", &[os(&git_dir), os("rev-parse"), os("HEAD")])?; + Ok(Proven { + head: v.head, + blobs: v.blobs, + commit, + }) + }); + let _ = std::fs::remove_dir_all(scratch); + proven +} + +/// The book's archive proven to hold the book as it is ([`prove_archive`]). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Proven { + /// The journal's head it holds: the book's. + pub head: (u64, String), + /// Stored files re-hashed. + pub blobs: u64, + /// The commit proven: the archive's latest. + pub commit: String, +} + +/// Every file under `rel` in `base`, by its path there, with its bytes +/// (none when `rel` isn't there). +fn held_files(base: &Path, rel: &str) -> Result)>, BackupError> { + if std::fs::symlink_metadata(base.join(rel)).is_err() { + return Ok(Vec::new()); + } + let mut names = Vec::new(); + files_under(base, Path::new(rel), &mut names)?; + names.sort(); + names + .into_iter() + .map(|n| { + let bytes = std::fs::read(base.join(&n))?; + Ok((n, bytes)) + }) + .collect() +} + /// Whether the journal at `backup` is a byte prefix of the one at `live` /// (a drill: the backup holds the book's history, nothing else). pub fn is_prefix(live: &Path, backup: &Path) -> std::io::Result { @@ -1471,6 +1555,59 @@ mod tests { ); } + /// The backup a format upgrade makes first (plan 3.9): the archive + /// proves to hold the book exactly as it is, its keys and vault file + /// for file; a key, a vault record or the journal that differs from + /// the archive's latest, or a head it doesn't hold, is refused; and + /// the scratch folder goes either way. + #[test] + fn the_archive_proves_to_hold_the_book_as_it_is_keys_and_all() { + let git = Git::find().expect("git is installed"); + let dir = tempfile::tempdir().unwrap(); + let b = dir.path().join("book"); + let mut s = book(&b); + let keys = b.join(crate::root::KEYS_DIR); + std::fs::create_dir(&keys).unwrap(); + std::fs::write(keys.join(crate::root::LEGACY_KEY_FILE), b"ab".repeat(32)).unwrap(); + let vault = b.join(crate::root::VAULT_DIR); + std::fs::create_dir(&vault).unwrap(); + std::fs::write(vault.join("tin-1.bin"), b"sealed").unwrap(); + let made = commit(&git, &b, "acme", s.head()).unwrap().unwrap(); + let scratch = dir.path().join("check"); + let proven = prove_archive(&git, &b, &s.head(), &scratch).unwrap(); + assert_eq!(proven.head, s.head()); + assert_eq!(proven.commit, made.commit); + assert_eq!(proven.blobs, 1); + assert!(!scratch.exists(), "the scratch folder is removed"); + let refused = |b: &Path, head: &(u64, String)| { + let e = prove_archive(&git, b, head, &scratch).unwrap_err(); + assert!(!scratch.exists()); + e.to_string() + }; + // A key that isn't the one backed up. + std::fs::write(keys.join(crate::root::LEGACY_KEY_FILE), b"cd".repeat(32)).unwrap(); + assert!(refused(&b, &s.head()).contains("keys folder")); + std::fs::write(keys.join(crate::root::LEGACY_KEY_FILE), b"ab".repeat(32)).unwrap(); + // A key the backup lacks. + std::fs::write(keys.join("other.key"), b"x").unwrap(); + assert!(refused(&b, &s.head()).contains("keys folder")); + std::fs::remove_file(keys.join("other.key")).unwrap(); + // A vault record that isn't the one backed up. + std::fs::write(vault.join("tin-1.bin"), b"sealed again").unwrap(); + assert!(refused(&b, &s.head()).contains("TIN vault")); + std::fs::write(vault.join("tin-1.bin"), b"sealed").unwrap(); + assert!(prove_archive(&git, &b, &s.head(), &scratch).is_ok()); + // The books moved on since the backup. + s.add_account("6000", "Fees", AccountType::Expense, None, false) + .unwrap(); + let e = refused(&b, &s.head()); + assert!(e.contains("they are at event"), "{e}"); + // The head it holds, but the journal on disk changed under it. + let old_head = made.head.clone(); + let e = refused(&b, &old_head); + assert!(e.contains("byte for byte"), "{e}"); + } + #[test] fn a_backup_holding_a_link_or_a_stranger_is_refused_before_checkout() { let git = Git::find().expect("git is installed"); diff --git a/crates/ledger-store/src/error.rs b/crates/ledger-store/src/error.rs index 6802eaf..5affe43 100644 --- a/crates/ledger-store/src/error.rs +++ b/crates/ledger-store/src/error.rs @@ -82,6 +82,18 @@ pub enum StoreError { #[error("{0}")] FormatRefused(String), + /// A write that needs a journal format or feature these books haven't + /// declared: they were written by an older version, and wait for the + /// owner's go-ahead to move to this version's format (the format + /// upgrade: plan/served.md 3.9, D-39). Nothing was written; everything + /// that needs nothing new still works. + #[error("{}", format_upgrade_text(need))] + FormatUpgradePending { + /// What the write needs: "journal format 2", "the journal + /// feature x". + need: String, + }, + /// Replay produced different ids than the journal recorded. #[error("replay mismatch: {0}")] ReplayMismatch(String), @@ -137,6 +149,24 @@ pub enum StoreError { CurrencyMismatch(String, String), } +/// The guard code a refusal for books waiting for their format upgrade +/// carries (plan/served.md 3.9: `format_upgrade_pending`). +pub const FORMAT_UPGRADE_PENDING: &str = "format_upgrade_pending"; + +/// What [`StoreError::FormatUpgradePending`] says: what the write needs, +/// that nothing was changed, that only the owner can give the go-ahead +/// (and where), and that the rest still works. +fn format_upgrade_text(need: &str) -> String { + format!( + "{FORMAT_UPGRADE_PENDING}: this needs {need}, and these books are still in the journal \ + format an older version of wanderbooks wrote: they are waiting for the owner's \ + go-ahead to upgrade to this version's format. Nothing was changed. Only the owner can \ + give it, with the owner passphrase: in the app (it asks when it opens the books) or \ + with `wanderbooks books upgrade ID` at a terminal; no tool can. Until then everything \ + that needs nothing new works as before" + ) +} + /// What [`StoreError::NewerJournal`] says. Only a line whose own hash was /// checked lets it say that line is intact; it never vouches for the /// lines after it, which this version can't read. diff --git a/crates/ledger-store/src/format.rs b/crates/ledger-store/src/format.rs index 6392315..e76757c 100644 --- a/crates/ledger-store/src/format.rs +++ b/crates/ledger-store/src/format.rs @@ -20,10 +20,16 @@ //! can't read, and say whether that line is intact. //! //! A line an earlier version can't read is announced by a feature -//! ([`KNOWN_FEATURES`]): the writer declares it (`journal.format_set`, -//! the features in force and the new one) just before the first line that -//! needs it, so the earlier reader stops at the declaration, and books -//! that never use the feature stay readable by it. +//! ([`KNOWN_FEATURES`]): the books declare it (`journal.format_set`) +//! before the first line that needs it, so the earlier reader stops at the +//! declaration. +//! +//! The other way round, books an older version wrote are in an older +//! format: this version reads them as they are, and writes nothing that +//! needs a format or feature they haven't declared until the owner says +//! go (the format upgrade, rules.md B-27, plan/served.md 3.9, D-39: +//! [`JournalFormat::upgrade`], [`current`]). New books declare this +//! version's whole format as they are created. use crate::error::StoreError; use crate::journal::{JournalLine, ReadLine}; @@ -35,11 +41,11 @@ pub const SUPPORTED_JOURNAL_FORMAT: u32 = 2; /// Optional journal features this version reads. Each marks lines an /// older version can't read: a new event type (which it would refuse as /// unknown), or a field or value on an event type it knows (which it -/// would misread silently). A writer declares the feature -/// (`journal.format_set`) before its first such line -/// ([`crate::Store`]'s `ensure_feature`), so the older version stops -/// there instead (rules.md B-27). Books that never use one stay readable -/// by versions before it. +/// would misread silently). The books declare it (`journal.format_set`) +/// before their first such line, so the older version stops there instead +/// (rules.md B-27). Every one is declared at once: by new books as they +/// are created, and by older books at their format upgrade ([`current`]); +/// until then a write that needs one is refused. pub const KNOWN_FEATURES: &[&str] = &[ MEMBERS_CUMULATIVE, FEATURE_ACCOUNT_RENAMED, @@ -59,16 +65,15 @@ pub const KNOWN_FEATURES: &[&str] = &[ /// rule, which can split it differently. pub const MEMBERS_CUMULATIVE: &str = "members_cumulative"; -/// The feature declared before the first `account.renamed` (P3, defect -/// 15). +/// Feature: `account.renamed`, a new event type (P3, defect 15). pub const FEATURE_ACCOUNT_RENAMED: &str = "account_renamed"; -/// The feature declared before the first `settings.set` naming -/// `policy.require_tax_category` (P3, defect 8). +/// Feature: a `settings.set` naming `policy.require_tax_category`, a key +/// an earlier version doesn't know (P3, defect 8). pub const FEATURE_REQUIRE_TAX_CATEGORY: &str = "require_tax_category"; -/// The feature a `settings.set` naming `key` needs declared first: a key -/// an earlier version doesn't know. +/// The feature a `settings.set` naming `key` needs the books to have +/// declared: a key an earlier version doesn't know. pub(crate) fn setting_feature(key: &str) -> Option<&'static str> { match key { "policy.require_tax_category" => Some(FEATURE_REQUIRE_TAX_CATEGORY), @@ -262,6 +267,122 @@ impl JournalFormat { } } +/// The journal format this version writes: the newest format it reads, +/// with every feature it knows, sorted. New books declare it as they are +/// created ([`crate::Store::create`]), and books in an older format move +/// to it in one step at their format upgrade +/// ([`crate::Store::upgrade_format`]). +pub fn current() -> JournalFormat { + let mut features: Vec = KNOWN_FEATURES.iter().map(|f| f.to_string()).collect(); + features.sort_unstable(); + features.dedup(); + JournalFormat { + format: SUPPORTED_JOURNAL_FORMAT, + features, + } +} + +/// A format upgrade books wait for: from the format they are in to this +/// version's ([`current`]). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FormatUpgrade { + /// The format the books are in. + pub from: JournalFormat, + /// The format the upgrade declares: this version's. + pub to: JournalFormat, +} + +impl FormatUpgrade { + /// What the upgrade adds, in words: a newer format number, then each + /// feature not yet declared. + pub fn adds(&self) -> Vec { + let mut out = Vec::new(); + if self.to.format > self.from.format { + out.push(format!("journal format {}", self.to.format)); + } + out.extend( + self.to + .features + .iter() + .filter(|f| !self.from.has_feature(f)) + .map(|f| format!("the journal feature {f}")), + ); + out + } +} + +impl JournalFormat { + /// Whether the books declared the feature `feature`. + pub fn has_feature(&self, feature: &str) -> bool { + self.features.iter().any(|f| f == feature) + } + + /// Refuse a write that needs journal format `format` (or a later one) + /// in books that haven't declared it: they wait for their format + /// upgrade (FormatUpgradePending; [`crate::Store`]'s writes ask this + /// before they journal anything). + pub fn require_format(&self, format: u32) -> Result<(), StoreError> { + if self.format >= format { + Ok(()) + } else { + Err(StoreError::FormatUpgradePending { + need: format!("journal format {format}"), + }) + } + } + + /// Refuse a write that needs the feature `feature` in books that + /// haven't declared it ([`Self::require_format`]'s kin). + pub fn require_feature(&self, feature: &str) -> Result<(), StoreError> { + if self.has_feature(feature) { + Ok(()) + } else { + Err(StoreError::FormatUpgradePending { + need: format!("the journal feature {feature}"), + }) + } + } + + /// The format upgrade books in this format wait for, if any (rules.md + /// B-27, plan/served.md 3.9, D-39). + /// + /// ```text + /// GOAL: books an older version wrote are never moved to a newer + /// format, which that version can't open, without the owner's + /// go-ahead, and whether they wait for one is a fact of their + /// journal alone. + /// SOURCE: plan/served.md 3.9 ("Format changes are the server's gate, + /// not the updater's": a server that would write a newer + /// journal format than a book has opens it on its old format + /// until the owner consents); D-39 (asked at every open until + /// given). + /// INVARIANTS: + /// 1. Some exactly when this version writes a higher format number, + /// or knows a feature the books haven't declared. + /// 2. Its `to` is this version's format (current()): every feature + /// it knows, so a book's format moves exactly once per version. + /// SCENARIOS: + /// 1. IF the books declared nothing (format 1, written before the + /// format gate): SHOULD wait for the upgrade to format 2. + /// 2. IF they declared this version's format and every feature it + /// knows: SHOULD wait for none. + /// 3. IF they declared this format without a feature this version + /// knows: SHOULD wait for one that adds it. + /// ``` + pub fn upgrade(&self) -> Option { + self.upgrade_to(current()) + } + + /// [`Self::upgrade`] for a version whose format is `to`. + fn upgrade_to(&self, to: JournalFormat) -> Option { + let behind = self.format < to.format || to.features.iter().any(|f| !self.has_feature(f)); + behind.then(|| FormatUpgrade { + from: self.clone(), + to, + }) + } +} + /// Read a `journal.format_set` line, refusing a format or feature this /// version doesn't read. pub(crate) fn declared(line: &JournalLine) -> Result { @@ -570,6 +691,136 @@ mod tests { ); } + /// This version's format: the newest it reads, every feature it + /// knows, each once, sorted (what new books and an upgrade declare). + #[test] + fn this_versions_format_is_the_newest_with_every_known_feature() { + let c = current(); + assert_eq!(c.format, SUPPORTED_JOURNAL_FORMAT); + let mut known: Vec = KNOWN_FEATURES.iter().map(|f| f.to_string()).collect(); + known.sort(); + known.dedup(); + assert_eq!(c.features, known); + // What it declares reads back as itself. + let features: Vec<&str> = c.features.iter().map(String::as_str).collect(); + let payload = JournalFormat::default() + .declaration(c.format, &features) + .unwrap(); + assert_eq!(declared(&line(FORMAT_SET, payload)).unwrap(), c); + assert_eq!(c.upgrade(), None, "this version's books wait for nothing"); + } + + /// Books wait for an upgrade exactly when this version writes a higher + /// format number or knows a feature they haven't declared; the upgrade + /// is always to this version's whole format (SCENARIOS 1 to 3). + #[test] + fn books_wait_for_an_upgrade_exactly_when_behind() { + let at = |format: u32, features: &[&str]| JournalFormat { + format, + features: features.iter().map(|f| f.to_string()).collect(), + }; + // Format 1 (nothing declared): waits for this version's format. + let old = JournalFormat::default(); + let up = old.upgrade().expect("format 1 books wait for format 2"); + assert_eq!((up.from.clone(), up.to.clone()), (old, current())); + assert_eq!( + up.adds()[0], + format!("journal format {SUPPORTED_JOURNAL_FORMAT}") + ); + // A version that knows features: each one missing makes books wait, + // and the upgrade adds all of them at once. + let to = at(2, &["a", "b"]); + for (books, adds) in [ + ( + at(1, &[]), + vec![ + "journal format 2", + "the journal feature a", + "the journal feature b", + ], + ), + ( + at(2, &[]), + vec!["the journal feature a", "the journal feature b"], + ), + (at(2, &["b"]), vec!["the journal feature a"]), + (at(2, &["a"]), vec!["the journal feature b"]), + ] { + let up = books.upgrade_to(to.clone()).expect("behind"); + assert_eq!(up.to, to, "the whole format, at once"); + assert_eq!(up.adds(), adds, "{books:?}"); + } + assert_eq!(at(2, &["a", "b"]).upgrade_to(to.clone()), None); + assert_eq!( + at(2, &["b", "a"]).upgrade_to(to.clone()), + None, + "in any order" + ); + // Nothing ahead of this version counts as behind (it never opens). + assert_eq!(at(3, &["a", "b"]).upgrade_to(to), None); + } + + /// A write that needs a format or feature the books haven't declared + /// is refused as waiting for the upgrade, saying what it needs; one + /// the books declared passes. + #[test] + fn a_write_needs_what_the_books_declared() { + let at = |format: u32, features: &[&str]| JournalFormat { + format, + features: features.iter().map(|f| f.to_string()).collect(), + }; + let need = |r: Result<(), StoreError>| match r { + Err(StoreError::FormatUpgradePending { need }) => need, + other => panic!("expected FormatUpgradePending, got {other:?}"), + }; + assert_eq!(need(at(1, &[]).require_format(2)), "journal format 2"); + at(2, &[]).require_format(2).unwrap(); + at(2, &[]).require_format(1).unwrap(); + assert_eq!( + need(at(2, &["a"]).require_feature("b")), + "the journal feature b" + ); + at(2, &["a", "b"]).require_feature("b").unwrap(); + // The refusal names the guard, says nothing changed, and that only + // the owner can agree, and how. + let text = StoreError::FormatUpgradePending { + need: "journal format 2".into(), + } + .to_string(); + for says in [ + "format_upgrade_pending:", + "needs journal format 2", + "Nothing was changed", + "Only the owner can give it", + "owner passphrase", + "wanderbooks books upgrade", + "no tool can", + ] { + assert!(text.contains(says), "{says}: {text}"); + } + } + + /// An upgrade never weakens the gate: a version that knows fewer + /// features than the books declared still stops at the declaration, + /// as newer, and one that knows them reads it. + #[test] + fn a_reader_that_doesnt_know_a_declared_feature_still_refuses_it_as_newer() { + let l = line(FORMAT_SET, json!({"format": 2, "features": ["a"]})); + match declared_knowing(&l, &[]) { + Err(StoreError::NewerJournal { need, .. }) => { + assert_eq!(need, "journal features a") + } + other => panic!("expected NewerJournal, got {other:?}"), + } + assert_eq!( + declared_knowing(&l, &["a"]).unwrap(), + JournalFormat { + format: 2, + features: vec!["a".into()] + } + ); + } + /// A format newer than this version reads is newer whatever shape the /// rest of its declaration takes, and however large its number: a /// newer format may define its features differently, so they are diff --git a/crates/ledger-store/src/lib.rs b/crates/ledger-store/src/lib.rs index 5440867..dd2422d 100644 --- a/crates/ledger-store/src/lib.rs +++ b/crates/ledger-store/src/lib.rs @@ -43,11 +43,11 @@ pub mod uploads; pub use blob::BlobStore; pub use error::StoreError; -pub use format::{JournalFormat, SUPPORTED_JOURNAL_FORMAT}; +pub use format::{FormatUpgrade, JournalFormat, SUPPORTED_JOURNAL_FORMAT}; pub use journal::{GENESIS_PREV, JournalLine, JournalWriter}; pub use snapshot::{DiskCheck, Snapshot}; pub use store::{ - Attachment, DB_FILE, DepreciationRun, DisposalPreview, EntryRow, EventRow, JOURNAL_FILE, + Attachment, DepreciationRun, DisposalPreview, EntryRow, EventRow, JOURNAL_FILE, LEGACY_DB_FILE, LOCK_FILE, MAX_RECURRING_RUN, PayrollLineInput, Reconciliation, Statement, Store, Subject, - VerifyReport, + VerifyReport, database_in_use, db_file, db_file_for, is_database_file, }; diff --git a/crates/ledger-store/src/root.rs b/crates/ledger-store/src/root.rs index b99e030..2d97a4e 100644 --- a/crates/ledger-store/src/root.rs +++ b/crates/ledger-store/src/root.rs @@ -665,7 +665,9 @@ impl Root { /// built in a staging folder: every stored file re-hashed against its /// name, the copy opened (its database rebuilt from the journal) with /// the source's head, every file it names present, verified end to - /// end, given its settings if the source keeps none (and verified + /// end, moved to this version's journal format if the source's is + /// older (the copy is new books of this version; the source keeps its + /// own), given its settings if the source keeps none (and verified /// again), synced, and only then renamed into place. pub fn adopt( &self, @@ -781,15 +783,22 @@ impl Root { } store.verify_and_heal()?; let blobs_verified = store.verify_blobs()?; + // The copy is new books of this version, in its journal format + // (the original, left as it was, keeps its own): it waits for + // no format upgrade, and its settings can move in. + let format_upgraded = store.upgrade_format()?.is_some(); let (settings_imported, settings_ignored) = match (&options.settings, keeps_settings) { (Some((changes, source)), false) => { let n = store.adopt_settings(changes, source)?.len(); - store.verify_and_heal()?; (n, false) } (Some(_), true) => (0, true), (None, _) => (0, false), }; + // Verified again for what was written on the way in. + if store.head() != head { + store.verify_and_heal()?; + } let final_head = store.head(); drop(store); sync_dir(&staging)?; @@ -806,6 +815,7 @@ impl Root { settings_imported, settings_ignored, source_lock_created, + format_upgraded, }) })(); let renamed = built.and_then(|report| { @@ -1660,9 +1670,9 @@ pub struct AdoptReport { pub id: BookId, /// The source's head, `(seq, hash)`, verified before copying. pub source_head: (u64, String), - /// The adopted books' head: the source's, plus (when settings were - /// moved in) their `settings.set`, after the journal format they need - /// if the source's is older. + /// The adopted books' head: the source's, plus this version's journal + /// format if the source's is older (`format_upgraded`), plus (when + /// settings were moved in) their `settings.set`. pub head: (u64, String), /// Stored files copied, each re-hashed against its name. pub blob_files: u64, @@ -1686,6 +1696,9 @@ pub struct AdoptReport { pub settings_ignored: bool, /// The source had no lock file, so taking its lock made one. pub source_lock_created: bool, + /// The copy was moved to this version's journal format (the source's + /// was older; the source keeps its own). + pub format_upgraded: bool, } /// The source's writer lock: its lock file opened read-only (so a diff --git a/crates/ledger-store/src/snapshot.rs b/crates/ledger-store/src/snapshot.rs index c10700a..c98e502 100644 --- a/crates/ledger-store/src/snapshot.rs +++ b/crates/ledger-store/src/snapshot.rs @@ -19,7 +19,7 @@ use crate::error::StoreError; use crate::format::JournalFormat; use crate::journal::{GENESIS_PREV, JournalLine}; use crate::store::{ - Attachment, BLOBS_DIR, DB_FILE, JOURNAL_FILE, Reconciliation, Statement, Store, Subject, + Attachment, BLOBS_DIR, JOURNAL_FILE, Reconciliation, Statement, Store, Subject, apply_line_to_sqlite, compare_projections, projection_rows, query_attachment_links, query_attachments, query_reconciliations, query_statements, read_attachment_bytes, replay_line_into_ledger, @@ -292,7 +292,8 @@ impl Snapshot { /// by the concurrency test, 2026-10-13, as a false "differs"). pub fn check_disk(&self) -> DiskCheck { use rusqlite::OpenFlags; - let path = self.data_dir.join(DB_FILE); + // The file this version's writer keeps (store::database_in_use). + let path = crate::store::database_in_use(&self.data_dir); if !path.exists() { return DiskCheck::Missing; } @@ -374,7 +375,7 @@ impl Snapshot { /// `file:` URI for `path`, read-only and immutable (sqlite.org/uri.html): /// every byte outside the unreserved set is percent-encoded. -fn immutable_uri(path: &Path) -> String { +pub(crate) fn immutable_uri(path: &Path) -> String { let mut out = String::from("file:"); for b in path.to_string_lossy().bytes() { if b.is_ascii_alphanumeric() || b"/-_.~".contains(&b) { diff --git a/crates/ledger-store/src/store.rs b/crates/ledger-store/src/store.rs index 4fbdedb..75f7590 100644 --- a/crates/ledger-store/src/store.rs +++ b/crates/ledger-store/src/store.rs @@ -52,8 +52,101 @@ use std::path::{Path, PathBuf}; /// File names inside the data directory. pub const JOURNAL_FILE: &str = "journal.jsonl"; -/// File names inside the data directory. -pub const DB_FILE: &str = "wanderbooks.db"; + +/// The projection database's one file before each schema had a file of +/// its own (every version through P2, whose schema was 12). This version +/// uses it only when it already holds this version's schema, so books that +/// need no migration aren't rebuilt for nothing; otherwise it is left as +/// it is, byte for byte, for the version that wrote it ([`db_file_for`]). +pub const LEGACY_DB_FILE: &str = "wanderbooks.db"; + +/// The projection database's file for schema `schema`: +/// `wanderbooks.v{schema}.db`. +/// +/// ```text +/// GOAL: going back to an older version always finds the books it can +/// open (plan/served.md 3.9: before consent, the way back is always +/// there), however far this version's projection has moved. +/// SOURCE: plan/served.md 3.9, the lead's decision (2026-10-21: each +/// schema in a file of its own); rules.md B-27. +/// INVARIANTS: +/// 1. A version opens or creates only its own schema's file (or the +/// legacy file when that already holds its schema); a missing one is +/// built from the journal. +/// 2. It never migrates, writes or deletes a file of another schema, +/// older or newer: each is left byte for byte for its own version. +/// 3. The name depends only on the schema number, so versions agree on +/// it whatever number each was given. +/// ``` +pub fn db_file_for(schema: u32) -> String { + format!("wanderbooks.v{schema}.db") +} + +/// This version's own projection file: [`db_file_for`] its schema +/// ([`crate::schema::SCHEMA_VERSION`]). +pub fn db_file() -> String { + db_file_for(crate::schema::SCHEMA_VERSION) +} + +/// Whether `name` is a projection database's file, of any schema (the +/// legacy one included): what [`Store::other_databases`] lists. +pub fn is_database_file(name: &str) -> bool { + name == LEGACY_DB_FILE + || name + .strip_prefix("wanderbooks.v") + .and_then(|rest| rest.strip_suffix(".db")) + .is_some_and(|n| !n.is_empty() && n.bytes().all(|b| b.is_ascii_digit())) +} + +/// The projection database this version uses for the books in `data_dir` +/// ([`db_file_for`] INVARIANT 1): its own schema's file if there is one; +/// else the legacy file if that already holds this version's schema; else +/// its own schema's file, to be built from the journal. Read only: a file +/// of another schema is looked at, never opened for writing. +pub fn database_in_use(data_dir: &Path) -> PathBuf { + let own = data_dir.join(db_file()); + if std::fs::symlink_metadata(&own).is_ok() { + return own; + } + let legacy = data_dir.join(LEGACY_DB_FILE); + if schema_of(&legacy) == Some(crate::schema::SCHEMA_VERSION) { + legacy + } else { + own + } +} + +/// The schema a projection file says it holds, read without writing a +/// byte of it (an immutable read when no writer has it open; None when it +/// isn't there, isn't a database, or says none). +pub(crate) fn schema_of(path: &Path) -> Option { + use rusqlite::OpenFlags; + if !path.is_file() { + return None; + } + let conn = if path.with_extension("db-wal").exists() { + Connection::open_with_flags( + path, + OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX, + ) + } else { + Connection::open_with_flags( + crate::snapshot::immutable_uri(path), + OpenFlags::SQLITE_OPEN_READ_ONLY + | OpenFlags::SQLITE_OPEN_NO_MUTEX + | OpenFlags::SQLITE_OPEN_URI, + ) + } + .ok()?; + conn.query_row( + "SELECT value FROM meta WHERE key='schema_version'", + [], + |r| r.get::<_, String>(0), + ) + .ok()? + .parse() + .ok() +} /// The lock file a writer holds while the books are open. Readers (the /// viewer's [`crate::Snapshot`]) never take it. @@ -250,10 +343,6 @@ pub struct Store { base_currency: Currency, /// The journal format in force (the format gate's record). format: JournalFormat, - /// Refuse every journal feature not yet in force - /// ([`Self::refuse_new_features`]): a stand-in for a format-upgrade - /// gate that refuses until the owner consents. - refuse_features: bool, } /// One filing of bytes into the blob store and journal. @@ -275,24 +364,44 @@ impl Store { /// Create a new store. Fails if one already exists. /// /// ```text - /// GOAL: start empty books in a data directory. - /// SOURCE: docs/receipt-store.md "Line format" (genesis prev); R-11. + /// GOAL: start empty books in a data directory, in this version's + /// journal format. + /// SOURCE: docs/receipt-store.md "Line format" (genesis prev); R-11; + /// plan/served.md 3.9 and D-39 (books this version makes wait + /// for no format upgrade). /// INVARIANTS: /// 1. The journal's first line MUST be `store.created` with the base /// currency, chained from the genesis hash (64 zeros). /// 2. The writer lock MUST be held from creation on. + /// 3. The second line declares this version's journal format, every + /// feature it knows (format::current()): the new books wait for + /// no format upgrade. /// SCENARIOS: /// 1. IF a journal already exists there: SHOULD refuse /// - NEVER overwrite or append to existing books. /// ``` pub fn create(data_dir: &Path, base_currency: Currency) -> Result { + Self::create_in_format(data_dir, base_currency, &crate::format::current()) + } + + /// [`Self::create`], declaring the journal format `format` (format 1, + /// [`JournalFormat::default`], declares nothing): books as an older + /// version made them, for tests of their format upgrade. Everything + /// else creates books with [`Self::create`]. + #[doc(hidden)] + pub fn create_in_format( + data_dir: &Path, + base_currency: Currency, + format: &JournalFormat, + ) -> Result { let journal_path = data_dir.join(JOURNAL_FILE); if journal_path.exists() { return Err(StoreError::AlreadyExists(data_dir.display().to_string())); } std::fs::create_dir_all(data_dir)?; let lock = lock_books(data_dir)?; - let db_path = data_dir.join(DB_FILE); + // New books: this version's own schema's file (db_file_for). + let db_path = data_dir.join(db_file()); let conn = Self::open_conn(&db_path)?; Self::migrate(&conn)?; Self::write_meta(&conn, "base_currency", base_currency.code())?; @@ -307,13 +416,16 @@ impl Store { ledger: Ledger::new(base_currency), base_currency, format: JournalFormat::default(), - refuse_features: false, }; let line = store.journal.append( "store.created", json!({ "base_currency": base_currency.code() }), )?; apply_line_to_sqlite(&store.conn, &line)?; + if *format != JournalFormat::default() { + let features: Vec<&str> = format.features.iter().map(String::as_str).collect(); + store.declare_journal_format(format.format, &features)?; + } Ok(store) } @@ -352,6 +464,13 @@ impl Store { /// for forensics and rebuild from the journal /// BECAUSE the database is derived; the journal is the record. /// 7. IF the database is behind the journal: SHOULD apply the rest. + /// 8. The database is this version's own schema's file, or the + /// legacy one when that already holds this version's schema + /// ([`database_in_use`]); a file of any other schema, older or + /// newer, is left byte for byte, never migrated, refused or + /// deleted; this version's own missing: SHOULD build it from the + /// journal (SCENARIO 5). One named as this version's that holds + /// another schema is damaged (SCENARIO 6). /// ``` pub fn open(data_dir: &Path) -> Result { let journal_path = data_dir.join(JOURNAL_FILE); @@ -385,12 +504,17 @@ impl Store { None }; let journal = JournalWriter::from_lines(&journal_path, &lines)?; - let db_path = data_dir.join(DB_FILE); + // This version's own schema's file (SCENARIO 8): another schema's + // is never opened for writing. + let db_path = database_in_use(data_dir); let mut fresh_db = !db_path.exists(); + let schema = crate::schema::SCHEMA_VERSION; let damage = if fresh_db { None } else if !Self::db_file_healthy(&db_path) { Some("failed integrity_check".to_string()) + } else if let Some(other) = schema_of(&db_path).filter(|v| *v != schema) { + Some(format!("holds schema {other}, not this version's {schema}")) } else { match Self::db_currency(&db_path) { Some(c) if c == base_currency => None, @@ -433,7 +557,6 @@ impl Store { ledger, base_currency, format, - refuse_features: false, }; store.catch_up(&lines)?; if let Some(t) = torn { @@ -579,9 +702,35 @@ impl Store { &self.format } + /// The projection database's file this version uses, by name + /// ([`database_in_use`]): what `books.verify` and `wanderbooks doctor` + /// say. + pub fn database_file(&self) -> String { + file_name_of(&self.db_path) + } + + /// The other projection files beside it (other schemas', the legacy + /// one if unused), by name, sorted: listed, never deleted + /// ([`db_file_for`] INVARIANT 2). + pub fn other_databases(&self) -> Vec { + let in_use = self.database_file(); + let mut out: Vec = std::fs::read_dir(&self.data_dir) + .into_iter() + .flatten() + .flatten() + .filter(|e| e.file_type().is_ok_and(|t| t.is_file())) + .map(|e| e.file_name().to_string_lossy().into_owned()) + .filter(|n| is_database_file(n) && *n != in_use) + .collect(); + out.sort(); + out + } + /// Declare the journal format: journal a `journal.format_set` line. /// A writer declares the format before the first event that needs it, - /// so an older reader stops at the declaration and says it is too old. + /// so an older reader stops at the declaration and says it is too old: + /// new books as they are created ([`Self::create`]), older books at + /// their format upgrade ([`Self::upgrade_format`]), never a tool. /// /// ```text /// GOAL: books that tell every reader which format they are in. @@ -613,74 +762,92 @@ impl Store { // items, approvals (journal format 2, crate::meta) // ------------------------------------------------------------------ - /// Declare journal format 2 before the first event that needs it, so - /// an older reader stops at the declaration (plan/served.md 5.1). - fn ensure_format(&mut self) -> Result<(), StoreError> { - if self.format.format < crate::meta::FORMAT { - let features = self.format.features.clone(); - let features: Vec<&str> = features.iter().map(String::as_str).collect(); - self.declare_journal_format(crate::meta::FORMAT, &features)?; - } - Ok(()) + /// Refuse an event that needs journal format 2 in books that haven't + /// declared it: books an older version wrote, waiting for their format + /// upgrade (the owner's go-ahead, [`Self::upgrade_format`]). Books this + /// version made declared it when they were created. + fn ensure_format(&self) -> Result<(), StoreError> { + self.format.require_format(crate::meta::FORMAT) } - /// Declare a journal feature (crate::format::KNOWN_FEATURES) before the - /// first line that needs it: a new event type, or a field or value on - /// a known event type that an older version would otherwise misread - /// silently. Declared with the format in force (at least format 2, - /// crate::meta::FORMAT) and every feature already in force, so an - /// older reader stops at the declaration (rules.md B-27); nothing is - /// journaled when it is already declared. Books that never use the - /// feature stay readable by older versions. + /// Refuse a write that needs the journal feature `feature` + /// (crate::format::KNOWN_FEATURES: a field or value on an event type an + /// older version knows, which that version would misread silently) + /// unless the books declared it. Books this version made declared it + /// when they were created; older books declare it only at their format + /// upgrade, with every other feature this version knows, never one at + /// a time and never without the owner's go-ahead + /// ([`Self::upgrade_format`]). Ask it only for a write that uses the + /// feature (the line about to be journaled holds the new field or + /// value): a write that doesn't is never refused, since declining the + /// upgrade keeps everything else working (D-39). /// - /// Each command here that writes a feature's line declares it first. - /// A caller that journals several commands as one (a tool drafting an + /// Each command here that writes a feature's line asks for it first. A + /// caller that journals several commands as one (a tool drafting an /// entry and tagging it) calls this before the first of them, once - /// every check has passed: a declaration never lands after another - /// line of the same command (where a refusal to upgrade the format - /// would leave the command half done), and a refused command leaves - /// none. + /// every check has passed: a refusal never lands after another line of + /// the same command (which would leave the command half done), and a + /// refused command leaves none. /// /// ```text /// GOAL: an older wanderbooks stops at the declaration (NewerJournal) - /// instead of misreading, or failing on, a line it can't read. - /// SOURCE: rules.md B-27; plan/served.md 5.1 (format gate). + /// instead of misreading a line it thinks it knows, and books + /// move past what their owner's older version reads only when + /// the owner says so. + /// SOURCE: rules.md B-27; plan/served.md 5.1 (format gate), 3.9 and + /// D-39 (the format upgrade). + /// SCENARIOS: + /// 1. IF the books declared it: SHOULD pass. + /// 2. IF not: SHOULD refuse (FormatUpgradePending), journaling + /// nothing. /// ``` - pub fn ensure_feature(&mut self, feature: &str) -> Result<(), StoreError> { - self.ensure_features(&[feature]) + pub fn ensure_feature(&self, feature: &str) -> Result<(), StoreError> { + debug_assert!( + crate::format::KNOWN_FEATURES.contains(&feature), + "{feature} isn't a feature this version knows" + ); + self.format.require_feature(feature) } - /// [`Self::ensure_feature`] for several at once: one declaration - /// naming every one not yet in force, or none when all are. - fn ensure_features(&mut self, needed: &[&str]) -> Result<(), StoreError> { - let mut features: Vec = self.format.features.clone(); - let before = features.len(); - for f in needed { - if !features.iter().any(|g| g == f) { - features.push(f.to_string()); - } - } - if features.len() == before { - return Ok(()); - } - if self.refuse_features { - return Err(StoreError::FormatRefused(format!( - "journal features {:?}: these books take no new feature now", - &features[before..] - ))); - } - let format = self.format.format.max(crate::meta::FORMAT); - let features: Vec<&str> = features.iter().map(String::as_str).collect(); - self.declare_journal_format(format, &features) + /// The format upgrade (plan/served.md 3.9, D-39): declare this + /// version's journal format, with every feature it knows, in one + /// `journal.format_set`, for books an older version wrote. What was + /// upgraded; None (journaling nothing) when the books already are in + /// this version's format. The books' figures don't change: only what + /// may be written from here on. + /// + /// Only the owner's go-ahead comes here: the books server, after the + /// owner passphrase and a verified backup with the keys (ledger-mcp + /// local.rs and backup.rs); and an adoption, whose copy is new books + /// of this version while the original keeps its own format (root.rs). + /// No tool does. + /// + /// ```text + /// GOAL: older books move to this version's format at once, on the + /// record, exactly once. + /// SOURCE: plan/served.md 3.9 ("Format changes are the server's + /// gate"); D-39; rules.md B-27. + /// INVARIANTS: + /// 1. One journal line declares the whole format: the format moves + /// exactly once, never feature by feature. + /// 2. Books already in this version's format: nothing is journaled. + /// 3. The ledger is unchanged: every report reads as before. + /// ``` + pub fn upgrade_format(&mut self) -> Result, StoreError> { + let Some(upgrade) = self.format.upgrade() else { + return Ok(None); + }; + let features: Vec<&str> = upgrade.to.features.iter().map(String::as_str).collect(); + self.declare_journal_format(upgrade.to.format, &features)?; + debug_assert_eq!(self.format, upgrade.to); + Ok(Some(upgrade)) } - /// Refuse (`refuse: true`) or take again every journal feature not yet - /// in force, as a format-upgrade gate does that waits for the owner's - /// consent: what a command does when [`Self::ensure_feature`] refuses - /// (it journals nothing). Not saved: the books open taking features. - #[doc(hidden)] - pub fn refuse_new_features(&mut self, refuse: bool) { - self.refuse_features = refuse; + /// [`Self::ensure_feature`] for each feature a write needs: refused + /// (FormatUpgradePending, journaling nothing) unless the books declared + /// every one. + fn ensure_features(&self, needed: &[&str]) -> Result<(), StoreError> { + needed.iter().try_for_each(|f| self.ensure_feature(f)) } /// Change settings (`settings.set`): all or nothing, each value @@ -719,9 +886,10 @@ impl Store { if changes.is_empty() && !(adopt && !self.ledger.settings().is_journaled()) { return Ok(changes); } - // A key an older version doesn't know needs its feature declared - // first (crate::format::setting_feature), in the one declaration - // that also brings the books to format 2. + // A key an older version doesn't know needs its feature + // (crate::format::setting_feature), and every settings.set format + // 2: refused, journaling nothing, in books that haven't declared + // them (their format upgrade does). let needed: Vec<&str> = changes .iter() .filter_map(|c| crate::format::setting_feature(&c.key)) @@ -1451,10 +1619,12 @@ impl Store { /// Name a party the creditor of a liability account the company owes /// it itself on, or stop (`on` false): ledger_core's /// `Ledger::set_party_creditor`. Journaled as `party.creditor_set - /// {id, account_id, on}`, after the journal declares the feature it - /// needs ([`crate::format::PARTY_CREDITORS`]) the first time, so an - /// older wanderbooks stops there and says it is too old instead of - /// misreading the books. + /// {id, account_id, on}`, which needs the journal feature + /// [`crate::format::PARTY_CREDITORS`], so an older wanderbooks stops at + /// the declaration and says it is too old instead of misreading the + /// books. Books that haven't declared it (an older version's, until + /// their format upgrade) refuse it, FormatUpgradePending, journaling + /// nothing. pub fn set_party_creditor( &mut self, id: ledger_core::PartyId, @@ -1820,8 +1990,10 @@ impl Store { /// 3. Nothing due (the month rounds to nothing, a later run caught it /// up, the life is over and posted) journals nothing: an entry /// can't carry a zero line (P3 depreciation review R10). - /// 4. A period run before and reversed is run again only after - /// journal feature month_reposting is declared. + /// 4. A period run before and reversed is run again only in books + /// that declare journal feature month_reposting (books an older + /// version wrote: after their format upgrade); refused before + /// anything is journaled otherwise. /// SCENARIOS: /// 1. IF the asset is unknown or disposed, the period malformed, or /// it ends before the asset was placed in service: SHOULD refuse. @@ -2344,8 +2516,8 @@ impl Store { )?; // The disposal month's run was reversed (else the final // depreciation would be in the disposal entry): its line again, - // which needs a journal feature, declared (or refused) before the - // first line is journaled. + // which needs a journal feature, asked for (and refused in books + // that haven't declared it) before the first line is journaled. if depreciation.is_some() && self.depreciation_period_was_run(asset_id, &preview.period)? { self.ensure_feature(crate::format::MONTH_REPOSTING)?; } @@ -2447,17 +2619,19 @@ impl Store { // Rehearsed whole: a doomed close journals nothing, not even the // Retained Earnings account creation below. self.rehearse(|l| l.close_period_to(id, to))?; + if matches!(to, CloseTo::Members(_) | CloseTo::MembersCumulative { .. }) { + // A close counted from its regime's start: an older version + // would replay it under the run rule (format gate, B-27). + // Refused before anything is journaled until the books + // declare it. + self.ensure_feature(crate::format::MEMBERS_CUMULATIVE)?; + } if matches!(to, CloseTo::RetainedEarnings { .. }) { // Retained Earnings must exist before the close so its creation // (if needed) is journaled as account.added; the close's own // ensure then no-ops and replay stays deterministic. self.ensure_control_account(&ledger_core::RETAINED_EARNINGS)?; } - if matches!(to, CloseTo::Members(_) | CloseTo::MembersCumulative { .. }) { - // A close counted from its regime's start: an older version - // would replay it under the run rule (format gate, B-27). - self.ensure_feature(crate::format::MEMBERS_CUMULATIVE)?; - } let mut next = self.ledger.clone(); let closing_entry = next.close_period_to(id, to)?; let entry_detail = match closing_entry { @@ -3315,10 +3489,11 @@ impl Store { } /// Register an amortization schedule. Journaled as `schedule.registered`, - /// with its `convention` (`daily` or `monthly_even`). The first daily - /// schedule declares journal feature `schedule_convention` before its - /// line: an older version reads every schedule as equal months, and - /// would refuse a daily one's postings as wrong (rather than as newer). + /// with its `convention` (`daily` or `monthly_even`). A daily schedule + /// needs journal feature `schedule_convention` (refused, journaling + /// nothing, in books that haven't declared it): an older version reads + /// every schedule as equal months, and would refuse a daily one's + /// postings as wrong (rather than as newer). pub fn register_schedule( &mut self, new: ledger_core::NewSchedule, @@ -3352,8 +3527,9 @@ impl Store { /// Post one month of a schedule. Journaled as `schedule.posted` with the /// whole entry, so a month is never half-posted or posted twice; replay /// re-derives the entry and checks it line by line. A month a reversal - /// un-posted posts again only after journal feature `month_reposting` is - /// declared: an older version would refuse it as posted twice. + /// un-posted posts again only in books that declare journal feature + /// `month_reposting` (refused, journaling nothing, otherwise): an older + /// version would refuse it as posted twice. pub fn post_schedule_month( &mut self, id: ledger_core::ScheduleId, @@ -3535,10 +3711,9 @@ impl Store { /// Give an account a new name (see `Ledger::rename_account`), with an /// optional reason. Journaled as `account.renamed`, keyed by the /// account's id (a code can change; an id can't), with its code, the - /// new name, the name it had and the reason. The first rename declares - /// the journal feature `account_renamed` - /// ([`crate::format::FEATURE_ACCOUNT_RENAMED`]), so an earlier version - /// stops at the declaration. + /// new name, the name it had and the reason. It needs the journal + /// feature `account_renamed` ([`crate::format::FEATURE_ACCOUNT_RENAMED`]), + /// so an earlier version stops at the declaration. /// /// ```text /// GOAL: rename an account, keeping the name it had in the journal. @@ -3549,8 +3724,9 @@ impl Store { /// 1. Only the account's name changes (Ledger::rename_account /// INVARIANT 1): every entry, line, hash and earlier journal line /// stays as it was. - /// 2. Rehearsed and checked first: a refusal journals nothing, not - /// even the feature's declaration. + /// 2. Rehearsed and checked first: a refusal journals nothing (and + /// books that haven't declared `account_renamed` refuse it, + /// FormatUpgradePending, until their format upgrade). /// ``` pub fn rename_account( &mut self, @@ -3659,7 +3835,9 @@ impl Store { /// 2. IF the core refuses it: SHOULD refuse, journaling nothing. /// 3. IF it changes no decision: SHOULD journal nothing but a /// changed W-9 flag (Store::set_vendor_w9_flag). - /// 4. Otherwise: SHOULD declare the feature once, then journal it. + /// 4. Otherwise: SHOULD journal it, in books that declare the feature + /// (refused, FormatUpgradePending, journaling nothing, in books + /// that don't). /// ``` pub fn set_vendor_1099( &mut self, @@ -4638,6 +4816,8 @@ impl Store { .map(|l| sha256_hex(l.to_canonical().as_bytes())) .unwrap_or_else(|| GENESIS_PREV.to_string()), healed, + database: self.database_file(), + other_databases: self.other_databases(), }) } @@ -6960,6 +7140,19 @@ pub struct VerifyReport { pub head: String, /// Why the projection was rebuilt, if it was. pub healed: Option, + /// The projection database's file verified: this version's + /// ([`Store::database_file`]). + pub database: String, + /// Other schemas' projection files beside it, left as they are + /// ([`Store::other_databases`]). + pub other_databases: Vec, +} + +/// A path's last part, as text. +fn file_name_of(path: &Path) -> String { + path.file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_default() } /// A projection of `lines` into a private in-memory database, built with diff --git a/crates/ledger-store/tests/asset_orders.rs b/crates/ledger-store/tests/asset_orders.rs index 8acf5da..b883e5c 100644 --- a/crates/ledger-store/tests/asset_orders.rs +++ b/crates/ledger-store/tests/asset_orders.rs @@ -33,7 +33,7 @@ use chrono::NaiveDate; use ledger_core::{AccountType, AssetStatus, Currency, Money, NewAsset}; -use ledger_store::{DB_FILE, JOURNAL_FILE, Store}; +use ledger_store::{JOURNAL_FILE, Store, db_file}; use proptest::prelude::*; use rust_decimal::Decimal; use std::path::{Path, PathBuf}; @@ -309,10 +309,10 @@ fn check_completion(dir: &Path, asset: u64, ops: &[Op]) { /// Copy the books (journal and projection) to `to`, closed. fn copy_books(from: &Path, to: &Path) { - for f in [JOURNAL_FILE, DB_FILE] { - let src = from.join(f); + for f in [JOURNAL_FILE.to_string(), db_file()] { + let src = from.join(&f); if src.exists() { - std::fs::copy(&src, to.join(f)).unwrap(); + std::fs::copy(&src, to.join(&f)).unwrap(); } } } diff --git a/crates/ledger-store/tests/asset_tags.rs b/crates/ledger-store/tests/asset_tags.rs index be23dd1..a25e8aa 100644 --- a/crates/ledger-store/tests/asset_tags.rs +++ b/crates/ledger-store/tests/asset_tags.rs @@ -5,7 +5,9 @@ //! written before tags existed, refusals journaling nothing, and corrupt //! `asset.serial_set` lines refused on open; and tax basis (B-25: //! `asset.tax_basis_set` {asset_id, rule, contributor_basis | null}, the -//! v10 migration, corrupt payloads). +//! v10 schema, corrupt payloads). A database at an older schema under this +//! version's file name is rebuilt, never migrated (each schema keeps a file +//! of its own: rules.md B-27, tests/schema_files.rs). //! //! Expectations come from the doc comments on `Store::set_asset_tag`, //! `Store::set_asset_serial`, `Store::open`, `Store::verify`, @@ -14,7 +16,7 @@ use chrono::NaiveDate; use ledger_core::{AccountType, AssetId, Currency, LedgerError, Money, NewAsset}; use ledger_store::journal::read_and_verify_journal; -use ledger_store::{DB_FILE, JOURNAL_FILE, JournalWriter, Store, StoreError}; +use ledger_store::{JOURNAL_FILE, JournalWriter, Store, StoreError, db_file}; use serde_json::{Value, json}; use std::path::{Path, PathBuf}; @@ -245,7 +247,7 @@ fn tags_and_serials_replay_on_reopen() { for rebuild in [false, true] { if rebuild { - std::fs::remove_file(dir.join(DB_FILE)).unwrap(); + std::fs::remove_file(dir.join(db_file())).unwrap(); } let s = Store::open(&dir).unwrap(); s.verify().unwrap(); @@ -327,7 +329,7 @@ fn verify_passes_after_many_retags_and_serial_changes() { fn books_written_before_tags_open_and_verify() { let (dir, s, _, x, y) = fixture("older"); drop(s); - let _ = std::fs::remove_file(dir.join(DB_FILE)); + let _ = std::fs::remove_file(dir.join(db_file())); let mut s = Store::open(&dir).expect("older books still open"); s.verify().unwrap(); assert_eq!(tag_of(&s, x), None); @@ -356,7 +358,7 @@ fn an_older_asset_registered_line_still_replays() { ) .unwrap(); drop(w); - let _ = std::fs::remove_file(dir.join(DB_FILE)); + let _ = std::fs::remove_file(dir.join(db_file())); let mut s = Store::open(&dir).expect("older books still open"); s.verify().unwrap(); let old = y + 1; @@ -372,12 +374,23 @@ fn an_older_asset_registered_line_still_replays() { assert_eq!(tag_of(&s, old).as_deref(), Some("FL-0001")); } +/// Whether this version's database was kept aside (and rebuilt) as the +/// books opened: `wanderbooks.v{N}.corrupt.*` beside it. +fn kept_aside(dir: &Path) -> bool { + let stem = db_file().trim_end_matches(".db").to_string(); + std::fs::read_dir(dir).unwrap().flatten().any(|e| { + e.file_name() + .to_string_lossy() + .starts_with(&format!("{stem}.corrupt.")) + }) +} + /// Downgrade the database to schema v8 (no assets.tag / assets.serial, no /// asset_tags_log, and none of v10's tax basis columns), as an older /// wanderbooks left it. fn downgrade_db_to_v8(dir: &Path) { downgrade_db_to_v9(dir); - let conn = rusqlite::Connection::open(dir.join(DB_FILE)).unwrap(); + let conn = rusqlite::Connection::open(dir.join(db_file())).unwrap(); conn.execute_batch( "ALTER TABLE assets DROP COLUMN tag; ALTER TABLE assets DROP COLUMN serial; @@ -388,13 +401,17 @@ fn downgrade_db_to_v8(dir: &Path) { .unwrap(); } -/// An existing v8 database migrates to v9 on open and verifies. +/// A database at schema v8 under this version's own file name (what an +/// older wanderbooks once migrated in place) is never migrated now: each +/// schema keeps a file of its own (rules.md B-27), so this one is damaged, +/// kept aside and rebuilt from the journal as the books open, and verifies. #[test] fn a_v8_database_migrates_and_verifies() { let (dir, s, _, x, y) = fixture("v8"); drop(s); downgrade_db_to_v8(&dir); let mut s = Store::open(&dir).expect("a v8 database opens"); + assert!(kept_aside(&dir), "rebuilt, not migrated"); s.verify().unwrap(); s.set_asset_tag(x, "FL-0001").unwrap(); s.set_asset_serial(y, Some("SN-Y")).unwrap(); @@ -406,8 +423,8 @@ fn a_v8_database_migrates_and_verifies() { assert_eq!(serial_of(&s, y).as_deref(), Some("SN-Y")); } -/// A v8 database behind a journal that has tag lines (Store::open SCENARIO -/// 7: apply the rest) migrates, applies them, and verifies. +/// A v8 database behind a journal that has tag lines is rebuilt from the +/// whole journal, the tag lines with it (as above), and verifies. #[test] fn a_v8_database_behind_a_journal_with_tags_catches_up() { let (dir, s, _, x, _) = fixture("v8-behind"); @@ -423,6 +440,7 @@ fn a_v8_database_behind_a_journal_with_tags_catches_up() { .unwrap(); drop(w); let s = Store::open(&dir).expect("opens and catches up"); + assert!(kept_aside(&dir), "rebuilt, not migrated"); s.verify().unwrap(); assert_eq!(tag_of(&s, x).as_deref(), Some("FL-0001")); assert_eq!(serial_of(&s, x).as_deref(), Some("SN-1")); @@ -449,7 +467,7 @@ fn an_asset_serial_set_line_with_a_non_string_serial_is_corrupt() { Ok(_) => panic!("serial {bad}: a corrupt asset.serial_set line opened"), } // Rebuilding from the journal alone refuses it too. - let _ = std::fs::remove_file(dir.join(DB_FILE)); + let _ = std::fs::remove_file(dir.join(db_file())); assert!(Store::open(&dir).is_err(), "serial {bad}: rebuild opened"); } } @@ -497,7 +515,7 @@ fn the_acquisition_link_replays_and_verifies() { assert_eq!(s.ledger().get_asset(x).unwrap().acquired_by, Some(e)); s.verify().unwrap(); drop(s); - std::fs::remove_file(dir.join(DB_FILE)).unwrap(); + std::fs::remove_file(dir.join(db_file())).unwrap(); let s = Store::open(&dir).unwrap(); assert_eq!(s.ledger().get_asset(x).unwrap().acquired_by, Some(e)); s.verify().unwrap(); @@ -542,7 +560,7 @@ fn a_description_change_replays_and_verifies() { assert_eq!(p["description"], "Laptop, contributed jointly"); s.verify().unwrap(); drop(s); - std::fs::remove_file(dir.join(DB_FILE)).unwrap(); + std::fs::remove_file(dir.join(db_file())).unwrap(); let s = Store::open(&dir).unwrap(); assert_eq!( s.ledger().get_asset(x).unwrap().description, @@ -673,7 +691,7 @@ fn tax_basis_replays_on_reopen() { drop(s); for rebuild in [false, true] { if rebuild { - std::fs::remove_file(dir.join(DB_FILE)).unwrap(); + std::fs::remove_file(dir.join(db_file())).unwrap(); } let s = Store::open(&dir).unwrap(); s.verify().unwrap(); @@ -694,7 +712,7 @@ fn tax_basis_replays_on_reopen() { /// Downgrade the database to schema v10 (none of v11's attachment links, /// statements, reconciliations, or payroll_lines.employee_id). fn downgrade_db_to_v10(dir: &Path) { - let conn = rusqlite::Connection::open(dir.join(DB_FILE)).unwrap(); + let conn = rusqlite::Connection::open(dir.join(db_file())).unwrap(); conn.execute_batch( "DROP TABLE attachment_links; DROP TABLE statements; @@ -706,7 +724,8 @@ fn downgrade_db_to_v10(dir: &Path) { .unwrap(); } -/// An existing v10 database migrates on open and verifies. +/// A database at schema v10 under this version's own file name is rebuilt +/// from the journal as the books open (as the v8 one above), and verifies. #[test] fn a_v10_database_migrates_and_verifies() { let (dir, mut s, _, x, _) = fixture("v10"); @@ -714,6 +733,7 @@ fn a_v10_database_migrates_and_verifies() { drop(s); downgrade_db_to_v10(&dir); let s = Store::open(&dir).expect("a v10 database opens"); + assert!(kept_aside(&dir), "rebuilt, not migrated"); s.verify().unwrap(); assert_eq!(tag_of(&s, x).as_deref(), Some("FL-0001")); } @@ -722,7 +742,7 @@ fn a_v10_database_migrates_and_verifies() { /// assets.contributor_basis), as an older wanderbooks left it. fn downgrade_db_to_v9(dir: &Path) { downgrade_db_to_v10(dir); - let conn = rusqlite::Connection::open(dir.join(DB_FILE)).unwrap(); + let conn = rusqlite::Connection::open(dir.join(db_file())).unwrap(); conn.execute_batch( "ALTER TABLE assets DROP COLUMN tax_rule; ALTER TABLE assets DROP COLUMN contributor_basis; @@ -731,7 +751,8 @@ fn downgrade_db_to_v9(dir: &Path) { .unwrap(); } -/// An existing v9 database migrates on open, verifies, and takes tax bases. +/// A database at schema v9 under this version's own file name is rebuilt +/// as the books open (as the v8 one above), verifies, and takes tax bases. #[test] fn a_v9_database_migrates_and_verifies() { use ledger_core::TaxBasisRule::*; @@ -740,6 +761,7 @@ fn a_v9_database_migrates_and_verifies() { drop(s); downgrade_db_to_v9(&dir); let mut s = Store::open(&dir).expect("a v9 database opens"); + assert!(kept_aside(&dir), "rebuilt, not migrated"); s.verify().unwrap(); assert_eq!(tag_of(&s, x).as_deref(), Some("FL-0001")); assert_eq!(tax_of(&s, x), None); @@ -752,8 +774,8 @@ fn a_v9_database_migrates_and_verifies() { assert_eq!(tax_of(&s, y), Some(tax(Carryover, Some("640.00")))); } -/// A v9 database behind a journal with a tax basis line migrates, applies -/// it (Store::open SCENARIO 7), and verifies. +/// A v9 database behind a journal with a tax basis line is rebuilt from +/// the whole journal, that line with it, and verifies. #[test] fn a_v9_database_behind_a_journal_with_tax_basis_catches_up() { use ledger_core::TaxBasisRule::*; @@ -768,6 +790,7 @@ fn a_v9_database_behind_a_journal_with_tax_basis_catches_up() { .unwrap(); drop(w); let s = Store::open(&dir).expect("opens and catches up"); + assert!(kept_aside(&dir), "rebuilt, not migrated"); s.verify().unwrap(); assert_eq!(tax_of(&s, x), Some(tax(Carryover, Some("300.00")))); } @@ -799,7 +822,7 @@ fn an_asset_tax_basis_set_line_with_a_bad_payload_is_corrupt() { Err(other) => panic!("{bad}: expected JournalCorrupt, got {other:?}"), Ok(_) => panic!("{bad}: a corrupt asset.tax_basis_set line opened"), } - let _ = std::fs::remove_file(dir.join(DB_FILE)); + let _ = std::fs::remove_file(dir.join(db_file())); match Store::open(&dir) { Err(StoreError::JournalCorrupt(_)) => {} Err(other) => panic!("{bad} rebuild: expected JournalCorrupt, got {other:?}"), @@ -835,7 +858,7 @@ fn an_asset_is_corrected_until_depreciation_is_posted() { assert_eq!(p["previous"]["cost"]["amount"], "1200.00"); s.verify().unwrap(); drop(s); - std::fs::remove_file(dir.join(DB_FILE)).unwrap(); + std::fs::remove_file(dir.join(db_file())).unwrap(); let mut s = Store::open(&dir).unwrap(); assert_eq!( s.ledger().get_asset(x).unwrap().cost.amount.to_string(), diff --git a/crates/ledger-store/tests/assets.rs b/crates/ledger-store/tests/assets.rs index 60d9103..2b7bebd 100644 --- a/crates/ledger-store/tests/assets.rs +++ b/crates/ledger-store/tests/assets.rs @@ -396,9 +396,12 @@ fn a_skipped_month_is_caught_up_in_the_month_run() { } /// P3 depreciation review R3: a run reversed within its month is -/// un-posted, so its month is due again; running it again declares journal -/// feature month_reposting first (an older version would ignore the second -/// line for the month). +/// un-posted, so its month is due again; running it again needs journal +/// feature month_reposting (an older version would ignore the second line +/// for the month), which new books declare as they are created, so it +/// journals no declaration of its own (books an older version wrote wait +/// for their format upgrade: format.rs +/// a_write_waiting_for_the_upgrade_journals_nothing). #[test] fn a_reversed_depreciation_month_can_be_run_again() { let (dir, mut store) = open_store(); @@ -409,14 +412,28 @@ fn a_reversed_depreciation_month_can_be_run_again() { store .reverse_entry(jan_entry, date(2026, 1, 31), "posted to the wrong asset") .unwrap(); - assert!(store.journal_format().features.is_empty()); + assert!( + store + .journal_format() + .has_feature(ledger_store::format::MONTH_REPOSTING) + ); + let declarations = |dir: &std::path::Path| { + std::fs::read_to_string(dir.join(ledger_store::JOURNAL_FILE)) + .unwrap() + .lines() + .filter(|l| l.contains("journal.format_set")) + .count() + }; + let before = declarations(dir.path()); let again = store.run_depreciation(x, "2026-01").unwrap(); assert!(!again.already_posted); assert_eq!(again.amount, dec("1000.00")); assert_ne!(again.entry_id, Some(jan_entry)); + assert_eq!(*store.journal_format(), ledger_store::format::current()); assert_eq!( - store.journal_format().features, - vec![ledger_store::format::MONTH_REPOSTING.to_string()] + declarations(dir.path()), + before, + "no declaration of its own" ); // Idempotent again: the new run is the month's. let twice = store.run_depreciation(x, "2026-01").unwrap(); @@ -430,7 +447,7 @@ fn a_reversed_depreciation_month_can_be_run_again() { // Reopened (the projection rebuilt from the journal, then caught up), // the same. drop(store); - std::fs::remove_file(dir.path().join(ledger_store::DB_FILE)).unwrap(); + std::fs::remove_file(dir.path().join(ledger_store::db_file())).unwrap(); let store = Store::open(dir.path()).unwrap(); store.verify().unwrap(); let mut store = store; @@ -631,7 +648,7 @@ fn a_run_reversed_after_its_month_is_caught_up_in_the_reversals_month() { ); assert_eq!(balance(&store, accum, date(2026, 1, 31)), dec("-1000.00")); assert_eq!(balance(&store, accum, date(2026, 2, 28)), dec("-2000.00")); - assert!(store.journal_format().features.is_empty()); + assert_eq!(*store.journal_format(), ledger_store::format::current()); store.verify().unwrap(); } @@ -854,7 +871,7 @@ fn a_run_reinstated_after_its_rerun_was_reversed_still_counts() { store.verify().unwrap(); // The same after the projection is rebuilt from the journal. drop(store); - std::fs::remove_file(dir.path().join(ledger_store::DB_FILE)).unwrap(); + std::fs::remove_file(dir.path().join(ledger_store::db_file())).unwrap(); Store::open(dir.path()).unwrap().verify().unwrap(); } @@ -1069,7 +1086,7 @@ fn a_disposal_whose_month_has_run_charges_what_is_still_due_in_its_entry() { .filter(|l| l.contains("asset.depreciation_posted") && l.contains("\"2026-06\"")) .count(); assert_eq!(junes, 1, "June keeps one run"); - assert!(store.journal_format().features.is_empty()); + assert_eq!(*store.journal_format(), ledger_store::format::current()); let on = date(2026, 6, 30); assert_eq!(balance(&store, accum, on), Decimal::ZERO); assert_eq!(balance(&store, expense, on), dec("5500.00")); diff --git a/crates/ledger-store/tests/attachments.rs b/crates/ledger-store/tests/attachments.rs index a693627..0b7a8e1 100644 --- a/crates/ledger-store/tests/attachments.rs +++ b/crates/ledger-store/tests/attachments.rs @@ -17,7 +17,7 @@ use ledger_core::arap::DocumentLine; use ledger_core::{AccountId, AccountType, Currency, EntryId, Money, NewAsset, Side}; use ledger_store::journal::read_and_verify_journal; use ledger_store::store::{Reconciliation, Statement, Subject}; -use ledger_store::{Attachment, DB_FILE, JOURNAL_FILE, Store, StoreError}; +use ledger_store::{Attachment, JOURNAL_FILE, Store, StoreError, db_file}; use rust_decimal::Decimal; use serde_json::{Value, json}; use std::path::{Path, PathBuf}; @@ -137,7 +137,7 @@ fn refused_nothing_journaled( fn delete_db(dir: &Path) { for suffix in ["", "-wal", "-shm", "-journal"] { - let _ = std::fs::remove_file(dir.join(format!("{DB_FILE}{suffix}"))); + let _ = std::fs::remove_file(dir.join(format!("{}{suffix}", db_file()))); } } diff --git a/crates/ledger-store/tests/fixtures/p2-books/journal.jsonl b/crates/ledger-store/tests/fixtures/p2-books/journal.jsonl new file mode 100644 index 0000000..93dd977 --- /dev/null +++ b/crates/ledger-store/tests/fixtures/p2-books/journal.jsonl @@ -0,0 +1,7 @@ +{"payload":{"base_currency":"USD"},"prev":"0000000000000000000000000000000000000000000000000000000000000000","seq":1,"ts":"2026-10-21T15:47:13Z","type":"store.created"} +{"payload":{"account_type":"Asset","code":"1000","currency":"USD","id":1,"name":"Cash","parent":null,"placeholder":false},"prev":"f7654c1569eb4937590b36306bc57c162d23109ec5e73cfe1eb2a25f2a56613e","seq":2,"ts":"2026-10-21T15:47:13Z","type":"account.added"} +{"payload":{"account_type":"Equity","code":"3000","currency":"USD","id":2,"name":"Capital","parent":null,"placeholder":false},"prev":"0365a319dcc4a38d8ea85a0aa18cffa6d3507c85f7999ec676e868e577f5c4a8","seq":3,"ts":"2026-10-21T15:47:13Z","type":"account.added"} +{"payload":{"date":"2026-03-01","description":"Capital in","id":1},"prev":"706e1ee9e5804f21858822f83e55cdb91e3b6f7b122929944adf78639239e963","seq":4,"ts":"2026-10-21T15:47:13Z","type":"entry.created"} +{"payload":{"account":1,"amount":"500","currency":"USD","id":1,"side":"debit"},"prev":"1a77b7d31e6394b71a2ea90ca1c059e82bffda1223ac009776ab94719a299b1b","seq":5,"ts":"2026-10-21T15:47:13Z","type":"entry.line_added"} +{"payload":{"account":2,"amount":"500","currency":"USD","id":1,"side":"credit"},"prev":"7b976b885597c115457545dfc2c1ba21df432796941acc8ecdf971254fa0c0e8","seq":6,"ts":"2026-10-21T15:47:13Z","type":"entry.line_added"} +{"payload":{"date":"2026-03-01","description":"Capital in","id":1,"lines":[{"account":1,"amount":"500","currency":"USD","side":"debit"},{"account":2,"amount":"500","currency":"USD","side":"credit"}],"reversal_of":null},"prev":"0bf713aa4db27c948592d5e772ea49414d95d068891fa525874e3eddfcf82796","seq":7,"ts":"2026-10-21T15:47:13Z","type":"entry.posted"} diff --git a/crates/ledger-store/tests/fixtures/p2-books/wanderbooks.db b/crates/ledger-store/tests/fixtures/p2-books/wanderbooks.db new file mode 100644 index 0000000000000000000000000000000000000000..6dfa79238bc785fddbf17f0e118a21d3642e4994 GIT binary patch literal 212992 zcmeI*?Qa~%eFyMco_HsoNS$PfUegMr6&abl=qT~z$c7~tDJ@?O!V+oIWNkHY7R%is zxz^t8X=jfSZMeu$PJjJogw+woA+#=X*;ghGJNg%!nwlOSmAdxtxzb8 z(f_Z}|LE%ldUGoJK;MuFO^ta>Fr~c~HYVr2i3(8x{r^=i%mv`X*zwjH+mejkmO2KYh>@k1j z5SMOreOG?gcOL7W;WMr`qIZ{iSJkq*u%H}H_{O%wOotgBvvkL9GoJYH%-xOV%4SpB zTzTbQQ%giGRy2*ZeeXcGEp2sev-w(cLt9(l)Yi7{-K$fiIGQe#kQx2P#_Ii*jR)H6 z%?H|Ibf9`{uUe({(laIX{z6{H63NjUiH|df%F|_4Rh1`(%;L7~I9$G{i7d$=k(F2` zVkr~(*k^6a_40CQ5v{h_e24NK%e&QOdUlL7*5Bi9JH40Da=UF`?--Pr)FHWV_+6e3 zkLXg!mD=ItbXlF9Rh~SQ26lVjwN1vO?@uQrqR`P8W|#YJi+v$EjEiOU(j}!gBQ=6z z<>9yKghd!QnxbvAhDyFt+r3a$FJD%Avq8HGv3R7$FD6dY5B~fo>fL=xX6Eo+d8Q8Y zY}ex9w+o50gn^&;9HQz|o%8&xD)%|+r|Gy<<92mfPY0EhI*CfH_sw%UYW(P`^ia1QFg z?w&8HZ#VKfuypG6Mmp@NCm&YJ>dcJt-K~KVm)r4kX$?NQpe^d*J-Z`t)7j~CJ#)|C zd3~4Za$nD*kdbEB^H|$FNFSOT&c3lr(C&z!}ONh_V@UpHpRN$ z7W4PqjQSZQ-ZVuQo4KQ5n%$PHeLB6(-Hbi%ur?8m+9t2Bwrt*UIgPZju^}8SJDzJY zX7MDDY$bbV%j%2sO0OsfQ`>s18@_Ltdo5~GJZM+3fNEGJR@Yu_zC*)W#@5>Unikum zKl;|*oGPhHmlV5yBGG7Vx^xif4d0(l9Y4O2ICS`VQS4M(pDd}@ROMm+kih^bwmFb* zr&7m@Z6OXPKMvB^n>!<0!{KGw8W{D}J`Y|e6RjZ#6Rm->NBt6yx7d}f&Gpqa+Ub6C zZBz7j{y}G`OBbOXh<3@(+nv+$%IYxajzn^}e3?k(M+Qm# zJ>%xB7lPEr^yTP0M?+g3)M&+DeMyJxEKH^+nPkvl=F4Eha` zF6ur?iQI*Oqbqr=gN&s$U0Y+3d7Z9D9f3!~kM8aaYMJadFy zB1Uz{}fB*y_009U<00Izz00ba# z3L3reweo|}8DW$5Ce4NX!s zZdNEy#p(|W)t^*=a9fB*y_009U<00Izz00bZa0SFvhfiI6$3e%Tn z3a2m757;j+FW)sfwr@DiM_t=Lpo#xy6^fr$Wqd=%^Su@E)1VY5=Ksg4KQB~2p%;8X z00Izz00bZa0SG_<0uX=z1R!t{1Wp&H74e8*tX!O)5YGa{{J&ED*8=^+7X%;x0SG_< z0uX=z1Rwwb2tWV=Uq}H>DHX0Bm{4Ve#kQm1;Qs&PLiN9^AO99|A!=qT08qEXRlBFU}~K{;tK*0fB*y_009U<00Izzz^^ON>y}G}t52@oVr}0$xE}nj z@ek%K!)J51=LWxye6BucF>ZQx$G6=!MGbyGIYrsl-0envPX5&L-0g3CFlU&i+im+Y z)My27=jivF(Y9vS^H|$FpkvT)JEH(@TjE^ucb(0BSdZ;|vp)B0?i~CAwYd-9t`9iU*)4h9Yys+>=cs5EG6(?owH!0Ojz1~c*R9NUe zx)ql5^`xEdA9O^K27lOIl({R5So}e|tQsJ)MP=PKT4JxTX3>TngK7#@B2kL;$uUf} z=Q%KQTp#j6eVa z5P$##AOHafKmY;|fB*z?1+e}b2~ICH&n zdcvG|>hupzZ;k(I{L-l(pE_6ke({X*tHLMBTgs<}?qx@0XH?|#S zaL#-lyq>$e(OlVVYMU#s+-qt<*kVP~Y)e~R+ibqp+|b_GSiQfp@j!dM`JhfQ7UQO8 zcYNDzYn#n?Hnp{N`oDGWo`~#pJ#)|COt%c5rK3%k`?}F`yKO%w@=$rYtg5Q=#E?nd zwjGDd7d4Sk88kYb`>bubo*v6cJfYd?v$}aHckDLf`g=Sp(?N#xjzQ;2iE!WWyF4Av z+P-&?mAXppaB{k=&dw@N9!dkdz3oV zb#GH!uI*kZtCue;y;+&GNSXKFTudadAN;u!SnJ(=DnNKwo~gq;+qHQ3?Ly)#Vc_RI zhiJZ3S-p&wNi8W|f%7W1>9|x?cXe682X@p!RBCt6m(;f#O2KYh>@k1j*gn&ZuJ6jv zy6k;)i*-C^+J@+0^hP@DsopowmDNT=Ioy>L=`GSB=MzBbcwrTU=5!ZsoM*TBY^{>aNYM=dreVke1R z{vIFHrr40vV*Z|+Q9pyR#x&(%6OCn26PCqen%$PHZn`mIGxoT{+BBk#+9t0pST^st zoLTYVFd8X6*JRA%Ng&ZmsPi#)!)}OEJ2#y0gJC}E+T4~6Bhk9V9*=deOMR6Xf9je^ zT%Ns6Ez8rl4Q>bHPi^mPS$%O{=@n&rv#rOv;roWU*P^5Gpuxrhs$rv9U3<0p4qXsr zY^|-YX|X*PqvlY1bE>2+T~h4+iA0P|Bd8d6!}n)X$B%C$4jq186g$<{Crj!zRe9Jy zWH2O)Z4TtysnqdeTZqHSkApP!=FZ5gqQlFwA7M0j_&j)>O!OB)nCK%od(^Lk@m9RD zwYk2!Mmybau5F6CbV+)A6kJ`&HX>ijiT#7NLz#{ZkTDJ| z)dJF0DAs3(eQBbG#iF@wWwf5WktBNr39jjwz1!9k8^}vN85}zv^_UI~#If|$dN;>| z3$8ms?-2A~B3IOnl@c~#;ONGdbHg#}W-4z~@u)-cX#CXOoqlg5Zon$F8>dR@-Pyc$ z5UNvCh~AAKUMvQ;8XpAROE3=dDE?F;pHb-OjGAs#cw1|$Z*DacHM`%06QTWvoCxh0 zbZOxyawt1d$1vHR>sZVi#KkI4qO|)5l%winJ<)&F>0Z|An!Zi<#5!Glnc{R}0Ie;I zmDD@ad5(0&VdxFBZWN6=TxL~^SAOHafKmY;|fB*y_009U<00JjYAejIESz-K>Qy&+9R(+>Bd-mIB*Qfq{>U&d- z$-k)lTZK>jeB$-f?DSX4e^&Zl@vn>D9=oLcwD9MJKR#-T(v6Zl`L5_o_ETsHsFix{7u*@2>Y|zbc=k9tIONQF!!y zDN!C_C>Fb5v5Q<{71j1i?Km%X>AnA4S-o;adGdAH@rdaOCZ~=E?`p|v-rqv>I)i!1 z*gVVd>auW-A-+X~BOm_DbuT)dFl(Jku!4dnvNO6dmuB(BvXpJZF{txT_jA#@mCOyO zV9s4FtIZ{)cc(uA9m^0+k3HmiIG7EGFNP>t6X~?PA+~37K0SzA43Dad?zyf-pKPo4 z^{WonzzGV(j#Gj3tFcC3j`6tk!Bu@53OMV)dg=ODwqVxVypfo9NTkWDXNsiu}b# zs90jsv!Btbp3EUDTHetgPbrPcm9o09p!8N`8pVv_fgY}AOMLiBB87=4k-#7z#urPa zoNLNm6j`agr_t&VO2S}u2(4G}Tt_Uy5sM$aZaWb$pQ_f0XtBMh&xycA3Z%AOd-CEsf+c>wcga1%j(suN-tcH7*-X6=20n$_g6B@PsHRGpJ30PE>BhOsCTH; zj9EEd-1m~+`5?6r)&5`BErDY=TA>mS85wyqD4)r60BAf>!IW-pKxKmY;|fB*y_009U<00Izz!0{8n@Bcr3Wr7qz z00Izz00bZa0SG_<0uX=z1ddt&_y0#NisL~50uX=z1Rwwb2tWV=5P$##j-LST|Bqjp zAVm;>00bZa0SG_<0uX=z1Rwx`qZYu={~xs|jt2nz{}fB*y_009U<00Izz00fSo z0Pg>fUzs395P$##AOHafKmY;|fB*y_0D+?xz|a35wJ44U0SG_<0uX=z1Rwwb2tWV= z5IBAUxc@(XWr7qz00Izz00bZa0SG_<0uX=z1ddt&>;I2h6vu-A1Rwwb2tWV=5P$## zAOHaf96tfv{~y0HL5d&%0SG_<0uX=z1Rwwb2tWV=M=gN+|DzVg@gM*J2tWV=5P$## zAOHafKmY>APXIsvfBeb>DS`k5AOHafKmY;|fB*y_009UbwE*V-k6IMRg8&2|009U< z00Izz00bZa0SFvF0nGm&zcN9JAOHafKmY;|fB*y_009U<00KuXfcgKU7RB)(009U< z00Izz00bZa0SG_<0>@7P^Z&=MOpqc7KmY;|fB*y_009U<00Izzz)=fe{{N^&aXbh> z00Izz00bZa0SG_<0uX?}@e{!P|M4pmqzD2KfB*y_009U<00Izz00ba#)B@uEzf%51 z;q0epOH+S0H9PsAXTE#pjfwv~{Sykr7X%;x0SG_<0w+YEx4T?cuU%7~jQhs6!&uw* z4s@f_@!WmG;X@z4c6Xz>vf0!&S6;c-)P^D#E1JgHBW-nUv-w(cLwjRm_5RAn1MT(Z zgE~b8TiaGYerh?Id$+?p&Md}LQKs!1Y1+J^;L(9ZViU)z)NU=6)YW;VV7D#ym_Kr; zzUfBScjaf@bldxsG(oI4Vjq?dx9eqfeqQ;`8CejqsMy;ZiNc6QgvBA7(oj`>NGDFV z3qhGvfwx`jAhT?^?{!VT>ydt}p+&BFEpT*o-j`Kvz!xh`kNLJZm&c4jv+mdCAZ|o; z4>w5F_8Zqq>fPDAvf<44sq3MFiQZlB&3?74&dw@_Q?gW|@aX$eqCBEdQ5X-|L4ZC~ z8q~vfv-+x0Qt6C_QkB|kwUYXcTX`As7*!F|c5GR-b;oWq%3M}xz2_~K)myieCuNz@ ztZlNwt|b7nwh$OY;jwNt-oK3;EkxRN*@QCdOH6q&b+&qcAgXJFvmua3wnAKe`qrs;OuzMhkDPF)WZ9hPpAKcanWqcAdiw js#3%J{|ULQLpC4)0SG_<0uX=z1Rwwb2tWV=BLw~rAN`&3 literal 0 HcmV?d00001 diff --git a/crates/ledger-store/tests/form1099.rs b/crates/ledger-store/tests/form1099.rs index 83cdc35..c9726db 100644 --- a/crates/ledger-store/tests/form1099.rs +++ b/crates/ledger-store/tests/form1099.rs @@ -1,8 +1,9 @@ //! 1099 decisions and payment methods in the journal (plan P3, defects -//! 12, 18 and 23; D-40): declared as features before their first line, -//! replayed and projected exactly, refused without a trace, and books -//! written before them read as they always meant. Every name is -//! fictional. +//! 12, 18 and 23; D-40): written only in books that declare their +//! features (this version's from creation, an older version's after the +//! format upgrade), replayed and projected exactly, refused without a +//! trace, and books written before them read as they always meant. Every +//! name is fictional. //! //! ```text //! GOAL: the decision ("no 1099" included) and how each payment was made @@ -17,9 +18,11 @@ use ledger_core::{ AccountType, Currency, Dated1099, Decision1099, ExemptReason, Form1099, InstrumentKind, Money, Side, }; -use ledger_store::format::{FORMAT_SET, PAYMENT_INSTRUMENTS, VENDOR_1099_DECISIONS}; +use ledger_store::format::{FORMAT_SET, PAYMENT_INSTRUMENTS, VENDOR_1099_DECISIONS, current}; use ledger_store::journal::read_and_verify_journal; -use ledger_store::{DB_FILE, JOURNAL_FILE, JournalWriter, Snapshot, Store, StoreError}; +use ledger_store::{ + JOURNAL_FILE, JournalFormat, JournalWriter, Snapshot, Store, StoreError, db_file, +}; use rust_decimal::Decimal; use serde_json::json; use std::path::{Path, PathBuf}; @@ -76,7 +79,13 @@ struct Books { } fn books(dir: &Path) -> Books { - let mut store = Store::create(dir, Currency::USD).unwrap(); + books_in(dir, ¤t()) +} + +/// [`books`] in the journal format `format`: format 1 +/// ([`JournalFormat::default`]) is books an older version wrote. +fn books_in(dir: &Path, format: &JournalFormat) -> Books { + let mut store = Store::create_in_format(dir, Currency::USD, format).unwrap(); store .add_account("1000", "Bank", AccountType::Asset, None, false) .unwrap(); @@ -130,26 +139,57 @@ fn types(dir: &Path) -> Vec<(String, serde_json::Value)> { .collect() } -/// B-27: the first line that needs a feature comes after a declaration of -/// it (with every feature already in force), once. A version that doesn't -/// know the feature stops at the declaration (`tests/format.rs` +/// B-27: a line that needs a feature is written only in books that +/// declare it, and a command never declares one on the way. Books an +/// older version wrote (format 1) refuse a decision and a payment method +/// while they wait for their format upgrade, journaling nothing; the +/// upgrade is one declaration of every feature; after it, as in books this +/// version made (which declared them when created), each command journals +/// its own lines and nothing else. A version that doesn't know a feature +/// stops at the declaration (`tests/format.rs` /// old_reader_gets_newer_journal, for any feature it doesn't know). #[test] -fn each_feature_is_declared_once_before_its_first_line() { +fn each_feature_waits_for_the_upgrade_and_is_never_declared_on_the_way() { let dir = test_dir("declare"); - let mut b = books(&dir); - let before = types(&dir).len(); + let mut b = books_in(&dir, &JournalFormat::default()); + let was = std::fs::read(dir.join(JOURNAL_FILE)).unwrap(); assert!( !types(&dir).iter().any(|(t, _)| t == FORMAT_SET), - "books without either feature declare nothing" + "books an older version wrote declare nothing" ); - b.store - .set_vendor_1099( - b.harbor, - &from_beginning(Decision1099::Files(Form1099::Nec)), + let nec = from_beginning(Decision1099::Files(Form1099::Nec)); + fn waits(r: Result, feature: &str) { + assert!( + matches!(&r, Err(StoreError::FormatUpgradePending { need }) if need.contains(feature)), + "{feature}: {r:?}" + ); + } + waits( + b.store.set_vendor_1099(b.harbor, &nec, None), + VENDOR_1099_DECISIONS, + ); + waits( + b.store.set_vendor_w9_flag(b.harbor, true), + VENDOR_1099_DECISIONS, + ); + waits( + b.store.add_instrument( + InstrumentKind::MemberCard, + "Riley's Visa", + Some("4242"), None, - ) - .unwrap(); + b.due, + ), + PAYMENT_INSTRUMENTS, + ); + assert_eq!( + std::fs::read(dir.join(JOURNAL_FILE)).unwrap(), + was, + "nothing journaled" + ); + let before = types(&dir).len(); + assert!(b.store.upgrade_format().unwrap().is_some()); + b.store.set_vendor_1099(b.harbor, &nec, None).unwrap(); b.store.set_vendor_w9_flag(b.harbor, true).unwrap(); let card = b .store @@ -171,26 +211,48 @@ fn each_feature_is_declared_once_before_its_first_line() { b.due, ) .unwrap(); - let added: Vec = types(&dir)[before..] - .iter() - .map(|(t, p)| match t.as_str() { - FORMAT_SET => format!("{t} {}", p["features"]), - _ => t.clone(), - }) - .collect(); + let j = types(&dir); + let added: Vec<&str> = j[before..].iter().map(|(t, _)| t.as_str()).collect(); assert_eq!( added, [ - format!("{FORMAT_SET} [\"{VENDOR_1099_DECISIONS}\"]"), - "vendor.1099_set".to_string(), - "vendor.w9_flag_set".to_string(), - format!("{FORMAT_SET} [\"{PAYMENT_INSTRUMENTS}\",\"{VENDOR_1099_DECISIONS}\"]"), - "instrument.added".to_string(), - "entry.instrument_set".to_string(), - "instrument.added".to_string(), + FORMAT_SET, + "vendor.1099_set", + "vendor.w9_flag_set", + "instrument.added", + "entry.instrument_set", + "instrument.added", ] ); - assert_eq!(b.store.journal_format().format, 2); + let declared = &j[before].1["features"]; + for feature in [VENDOR_1099_DECISIONS, PAYMENT_INSTRUMENTS] { + assert!( + declared.as_array().unwrap().contains(&json!(feature)), + "the upgrade declares {feature}" + ); + } + assert_eq!(b.store.journal_format(), ¤t()); + b.store.verify().unwrap(); + + // Books this version made: declared at creation, so the same commands + // journal only their own lines. + let dir = test_dir("declare-new"); + let mut b = books(&dir); + let before = types(&dir).len(); + b.store.set_vendor_1099(b.harbor, &nec, None).unwrap(); + b.store + .add_instrument( + InstrumentKind::MemberCashOrCheck, + "Riley's checks", + None, + None, + b.due, + ) + .unwrap(); + let j = types(&dir); + let added: Vec<&str> = j[before..].iter().map(|(t, _)| t.as_str()).collect(); + assert_eq!(added, ["vendor.1099_set", "instrument.added"]); + assert_eq!(j.iter().filter(|(t, _)| t == FORMAT_SET).count(), 1); } /// Every record replays and projects exactly: reopening gives back the @@ -252,14 +314,14 @@ fn decisions_and_methods_replay_and_project() { assert!(live.differs_from(store.ledger()).is_empty()); store.verify().unwrap(); drop(store); - std::fs::remove_file(dir.join(DB_FILE)).unwrap(); + std::fs::remove_file(dir.join(db_file())).unwrap(); let store = Store::open(&dir).unwrap(); store.verify().unwrap(); assert!(live.differs_from(store.ledger()).is_empty()); drop(store); let snap = Snapshot::load(&dir).unwrap(); assert!(live.differs_from(snap.ledger()).is_empty()); - let conn = rusqlite::Connection::open(dir.join(DB_FILE)).unwrap(); + let conn = rusqlite::Connection::open(dir.join(db_file())).unwrap(); let decisions: Vec = conn .prepare("SELECT effective, form_1099, reason_kind, source_json FROM vendor_1099_decisions ORDER BY seq") .unwrap() @@ -478,7 +540,7 @@ fn an_unknown_1099_value_is_newer_not_corrupt() { .unwrap(); let seq = w.append(t, payload).unwrap().seq; drop(w); - std::fs::remove_file(dir.join(DB_FILE)).unwrap(); + std::fs::remove_file(dir.join(db_file())).unwrap(); match (Store::open(&dir), need) { ( Err(StoreError::NewerJournal { @@ -502,7 +564,7 @@ fn an_unknown_1099_value_is_newer_not_corrupt() { #[test] fn books_written_before_decisions_read_as_they_always_meant() { let dir = test_dir("legacy"); - let b = books(&dir); + let b = books_in(&dir, &JournalFormat::default()); let (harbor, paid) = (b.harbor, b.paid); drop(b); let mut w = JournalWriter::open(&dir.join(JOURNAL_FILE)).unwrap(); @@ -519,7 +581,7 @@ fn books_written_before_decisions_read_as_they_always_meant() { w.append("vendor.1099_set", p).unwrap(); } drop(w); - std::fs::remove_file(dir.join(DB_FILE)).unwrap(); + std::fs::remove_file(dir.join(db_file())).unwrap(); let journal = std::fs::read(dir.join(JOURNAL_FILE)).unwrap(); let store = Store::open(&dir).unwrap(); store.verify().unwrap(); @@ -543,7 +605,7 @@ fn books_written_before_decisions_read_as_they_always_meant() { assert_eq!(store.journal_format().features, Vec::::new()); drop(store); assert_eq!(std::fs::read(dir.join(JOURNAL_FILE)).unwrap(), journal); - let conn = rusqlite::Connection::open(dir.join(DB_FILE)).unwrap(); + let conn = rusqlite::Connection::open(dir.join(db_file())).unwrap(); let forms: Vec<(i64, Option, bool)> = conn .prepare("SELECT id, form_1099, tin_on_file FROM vendors ORDER BY id") .unwrap() diff --git a/crates/ledger-store/tests/format.rs b/crates/ledger-store/tests/format.rs index 375a4b7..0b59620 100644 --- a/crates/ledger-store/tests/format.rs +++ b/crates/ledger-store/tests/format.rs @@ -20,8 +20,8 @@ use ledger_core::{AccountType, Currency, Money, Side}; use ledger_store::format::{EVENT_TYPES, FORMAT_SET, KNOWN_FEATURES}; use ledger_store::journal::{canonical_json, sha256_hex}; use ledger_store::{ - DB_FILE, JOURNAL_FILE, JournalFormat, JournalWriter, SUPPORTED_JOURNAL_FORMAT, Snapshot, Store, - StoreError, + JOURNAL_FILE, JournalFormat, JournalWriter, SUPPORTED_JOURNAL_FORMAT, Snapshot, Store, + StoreError, db_file, }; use rust_decimal::Decimal; use serde_json::json; @@ -57,9 +57,11 @@ fn test_dir(name: &str) -> TestDir { TestDir(dir) } -/// Books with one posted entry, closed again. +/// Books with one posted entry, closed again: in journal format 1, as +/// every version before the format gate wrote them (this version's own +/// declare its format as they are created). fn books(dir: &Path) { - let mut store = Store::create(dir, Currency::USD).unwrap(); + let mut store = Store::create_in_format(dir, Currency::USD, &JournalFormat::default()).unwrap(); let cash = store .add_account("1000", "Cash", AccountType::Asset, None, false) .unwrap(); @@ -177,12 +179,12 @@ fn old_reader_gets_newer_journal() { ), ], )[0]; - std::fs::remove_file(dir.join(DB_FILE)).unwrap(); + std::fs::remove_file(dir.join(db_file())).unwrap(); let journal = std::fs::read(dir.join(JOURNAL_FILE)).unwrap(); assert_newer(Store::open(&dir), seq, line_type, need, true); assert!( - !dir.join(DB_FILE).exists(), + !dir.join(db_file()).exists(), "a refused open rebuilt the database" ); assert_eq!(std::fs::read(dir.join(JOURNAL_FILE)).unwrap(), journal); @@ -746,7 +748,7 @@ fn a_bad_complete_line_before_a_torn_tail_leaves_the_journal_as_it_was() { append_raw(dir, b"{\"seq\":"); let path = dir.join(JOURNAL_FILE); let before = std::fs::read(&path).unwrap(); - let had_db = dir.join(ledger_store::DB_FILE).exists(); + let had_db = dir.join(ledger_store::db_file()).exists(); assert!(Store::open(dir).is_err(), "{}", dir.display()); assert_eq!(std::fs::read(&path).unwrap(), before, "{}", dir.display()); let set_aside: Vec<_> = std::fs::read_dir(dir) @@ -756,7 +758,7 @@ fn a_bad_complete_line_before_a_torn_tail_leaves_the_journal_as_it_was() { .collect(); assert!(set_aside.is_empty(), "{set_aside:?}"); assert_eq!( - dir.join(ledger_store::DB_FILE).exists(), + dir.join(ledger_store::db_file()).exists(), had_db, "no database made for books that can't open" ); @@ -885,16 +887,16 @@ fn read_journal_and_format_gives_every_line_and_the_format() { } /// P3's journal extensions (rules.md B-27; plan D-39): each is announced -/// by a feature this version knows, declared once, just before the first -/// line that needs it, with the features in force kept: `account_renamed` -/// before the first `account.renamed`, `require_tax_category` before the -/// first `settings.set` naming `policy.require_tax_category`. A reader -/// whose features don't include it (every earlier version: none) stops at -/// the declaration as newer, never at the line, and never calls it -/// corrupt; books that never use either stay format 1 or 2 with no -/// feature. +/// by a feature this version knows, `account_renamed` for an +/// `account.renamed`, `require_tax_category` for a `settings.set` naming +/// `policy.require_tax_category`. Books an older version made (books(): +/// format 1, nothing declared) refuse both, journaling nothing, until the +/// owner's format upgrade declares this version's whole format at once; +/// then both are taken, under that one declaration. A reader whose +/// features don't include them (every earlier version: none) stops at the +/// declaration as newer, never at the line, and never calls it corrupt. #[test] -fn p3_features_are_declared_once_before_the_first_line_that_needs_them() { +fn p3_features_wait_for_the_upgrade_and_are_declared_once() { for f in ["account_renamed", "require_tax_category"] { assert!(KNOWN_FEATURES.contains(&f), "{f}"); } @@ -902,26 +904,35 @@ fn p3_features_are_declared_once_before_the_first_line_that_needs_them() { books(&dir); let mut store = Store::open(&dir).unwrap(); assert!(store.journal_format().features.is_empty()); + let policy = || ledger_core::settings::Change { + key: "policy.require_tax_category".into(), + value: Some(json!("refuse")), + }; + let source = ledger_core::settings::Source { + decided_by: "Owner".into(), + decided_on: NaiveDate::from_ymd_opt(2026, 10, 20).unwrap(), + citation: None, + note: None, + }; + let head = store.head(); + let waits = |r: Result<(), StoreError>, need: &str| match r { + Err(StoreError::FormatUpgradePending { need: n }) => assert_eq!(n, need), + other => panic!("expected FormatUpgradePending, got {other:?}"), + }; + waits( + store.rename_account(1, "Business Checking", None), + "the journal feature account_renamed", + ); + waits( + store.change_settings(&[policy()], &source).map(|_| ()), + "the journal feature require_tax_category", + ); + assert_eq!(store.head(), head, "refused whole"); + assert!(store.upgrade_format().unwrap().is_some()); store.rename_account(1, "Business Checking", None).unwrap(); store.rename_account(1, "Checking", None).unwrap(); - store - .change_settings( - &[ledger_core::settings::Change { - key: "policy.require_tax_category".into(), - value: Some(json!("refuse")), - }], - &ledger_core::settings::Source { - decided_by: "Owner".into(), - decided_on: NaiveDate::from_ymd_opt(2026, 10, 20).unwrap(), - citation: None, - note: None, - }, - ) - .unwrap(); - assert_eq!( - store.journal_format().features, - ["account_renamed", "require_tax_category"] - ); + store.change_settings(&[policy()], &source).unwrap(); + assert_eq!(*store.journal_format(), ledger_store::format::current()); drop(store); let text = std::fs::read_to_string(dir.join(JOURNAL_FILE)).unwrap(); let lines: Vec = text @@ -933,23 +944,19 @@ fn p3_features_are_declared_once_before_the_first_line_that_needs_them() { .map(|l| (l["type"].as_str().unwrap(), l["payload"].clone())) .filter(|(t, _)| matches!(*t, FORMAT_SET | "account.renamed" | "settings.set")) .collect(); + let current = ledger_store::format::current(); assert_eq!(kinds[0].0, FORMAT_SET); assert_eq!( kinds[0].1, - json!({"format": 2, "features": ["account_renamed"]}) + json!({"format": current.format, "features": current.features}) ); assert_eq!(kinds[1].0, "account.renamed"); assert_eq!(kinds[2].0, "account.renamed"); - assert_eq!(kinds[3].0, FORMAT_SET); - assert_eq!( - kinds[3].1, - json!({"format": 2, "features": ["account_renamed", "require_tax_category"]}) - ); - assert_eq!(kinds[4].0, "settings.set"); - assert_eq!(kinds.len(), 5); + assert_eq!(kinds[3].0, "settings.set"); + assert_eq!(kinds.len(), 4); // An earlier reader knows no feature (P2: KNOWN_FEATURES was empty), - // so the gate stops it at the first declaration, before the rename: - // the gate's refusal of a feature it doesn't know is pinned by + // so the gate stops it at the declaration, before the rename: the + // gate's refusal of a feature it doesn't know is pinned by // an_unknown_feature_before_a_rename_is_newer_at_the_declaration. // This version opens them, in the writer and the viewer. let store = Store::open(&dir).unwrap(); @@ -980,7 +987,7 @@ fn an_unknown_feature_before_a_rename_is_newer_at_the_declaration() { ), ], ); - let _ = std::fs::remove_file(dir.join(DB_FILE)); + let _ = std::fs::remove_file(dir.join(db_file())); match Store::open(&dir) { Err(StoreError::NewerJournal { seq, need, .. }) => { assert_eq!(seq, seqs[0]); @@ -1034,33 +1041,41 @@ fn lines(dir: &Path) -> Vec { .collect() } -/// P3 (defect 9; rules.md B-27): the first daily schedule declares journal -/// feature schedule_convention, on the line before its schedule.registered, -/// which carries convention "daily". A version before P3 knew no features, -/// so it stops at that declaration (NewerJournal; format.rs's -/// an_older_reader_stops_at_p3s_features), never reading the schedule as -/// equal months. This version reads it back, posts by the day, and -/// verifies; a second daily schedule declares nothing more. +/// P3 (defect 9; rules.md B-27, D-39): a daily schedule's +/// schedule.registered, which carries convention "daily", needs journal +/// feature schedule_convention. Books an older version made (books(): +/// format 1) refuse one, journaling nothing, until the owner's format +/// upgrade, whose one declaration is on the line before it. A version +/// before P3 knew no features, so it stops at that declaration +/// (NewerJournal; format.rs's an_older_reader_stops_at_p3s_features), +/// never reading the schedule as equal months. This version reads it back, +/// posts by the day, and verifies; a second daily schedule declares +/// nothing more. #[test] -fn the_first_daily_schedule_declares_its_feature_and_an_older_reader_stops_there() { +fn a_daily_schedule_waits_for_its_feature_and_an_older_reader_stops_there() { let dir = test_dir("daily-feature"); let accounts = schedule_books(&dir); let mut store = Store::open(&dir).unwrap(); - let id = store - .register_schedule(insurance( - accounts, - day(2026, 3, 15), - ledger_core::ScheduleConvention::Daily, - )) - .unwrap(); + let daily = |start| insurance(accounts, start, ledger_core::ScheduleConvention::Daily); + let head = store.head(); + match store.register_schedule(daily(day(2026, 3, 15))) { + Err(StoreError::FormatUpgradePending { need }) => { + assert_eq!(need, "the journal feature schedule_convention") + } + other => panic!("expected FormatUpgradePending, got {other:?}"), + } + assert_eq!(store.head(), head, "refused whole"); + store.upgrade_format().unwrap(); + let id = store.register_schedule(daily(day(2026, 3, 15))).unwrap(); let journal = lines(&dir); let n = journal.len(); assert_eq!(journal[n - 1]["type"], "schedule.registered"); assert_eq!(journal[n - 1]["payload"]["convention"], "daily"); assert_eq!(journal[n - 2]["type"], FORMAT_SET); + let current = ledger_store::format::current(); assert_eq!( journal[n - 2]["payload"], - json!({"format": 2, "features": ["schedule_convention"]}) + json!({"format": current.format, "features": current.features}) ); store.run_schedules(day(2026, 5, 31)).unwrap(); store @@ -1152,7 +1167,7 @@ fn an_unknown_schedule_convention_is_newer_not_corrupt() { &dir, &[("schedule.registered", registered(json!("weekly")))], )[0]; - std::fs::remove_file(dir.join(DB_FILE)).unwrap(); + std::fs::remove_file(dir.join(db_file())).unwrap(); for got in [Store::open(&dir).err(), Snapshot::load(&dir).err()] { match got { Some(StoreError::NewerJournal { seq: s, need, .. }) => { @@ -1166,20 +1181,21 @@ fn an_unknown_schedule_convention_is_newer_not_corrupt() { let accounts2 = schedule_books(&dir); assert_eq!(accounts2, accounts); append(&dir, &[("schedule.registered", registered(json!(7)))]); - std::fs::remove_file(dir.join(DB_FILE)).unwrap(); + std::fs::remove_file(dir.join(db_file())).unwrap(); assert_corrupt(Store::open(&dir)); } /// P3 (R3; rules.md B-27): a schedule month reversed within its month is -/// posted again only after journal feature month_reposting is declared +/// posted again only in books that declare journal feature month_reposting /// (an older version would refuse the second schedule.posted as posted -/// twice), and the books replay to the same ledger. A month reversed after -/// it ended is left by the run, listed with why (it would count twice until -/// the reversal), and declares nothing. Hand check: equal months of 100.00 +/// twice): books an older version made wait for the format upgrade, and +/// the books replay to the same ledger. A month reversed after it ended is +/// left by the run, listed with why (it would count twice until the +/// reversal), and journals nothing. Hand check: equal months of 100.00 /// from 2026-03-01; March posted, reversed on Mar 31, run again: one /// March, 100.00; April posted, reversed on May 2: April held. #[test] -fn a_schedule_month_posted_again_declares_month_reposting_and_replays() { +fn a_schedule_month_posted_again_waits_for_month_reposting_and_replays() { let dir = test_dir("schedule-repost"); let accounts = schedule_books(&dir); let mut store = Store::open(&dir).unwrap(); @@ -1194,12 +1210,22 @@ fn a_schedule_month_posted_again_declares_month_reposting_and_replays() { store .reverse_entry(mar, day(2026, 3, 31), "posted early") .unwrap(); + // Books an older version made (books(): format 1) wait for the + // upgrade before March posts again; the owner's go-ahead declares it. assert!(store.journal_format().features.is_empty()); + let head = store.head(); + assert!(matches!( + store.run_schedules(day(2026, 3, 31)), + Err(StoreError::FormatUpgradePending { need }) if need == "the journal feature month_reposting" + )); + assert_eq!(store.head(), head, "refused whole"); + store.upgrade_format().unwrap(); let posted = store.run_schedules(day(2026, 3, 31)).unwrap(); assert_eq!(posted.len(), 1); - assert_eq!( - store.journal_format().features, - vec![ledger_store::format::MONTH_REPOSTING.to_string()] + assert!( + store + .journal_format() + .has_feature(ledger_store::format::MONTH_REPOSTING) ); let apr = store.post_schedule_month(id, day(2026, 4, 30)).unwrap(); store @@ -1228,18 +1254,19 @@ fn a_schedule_month_posted_again_declares_month_reposting_and_replays() { ); } -/// P3 review round 2 (the format-upgrade gate refuses a feature until the -/// owner consents): every command that needs a journal feature decides it, -/// and is refused, before its first journaled line, so a refused feature -/// journals nothing: never a depreciation entry without the +/// P3 review round 2, and the format-upgrade gate (D-39): in books an +/// older version wrote, every command that needs a journal feature is +/// refused (FormatUpgradePending) before its first journaled line, so the +/// refusal journals nothing: never a depreciation entry without the /// `asset.depreciation_posted` that makes it the period's run, nor a /// disposal's final depreciation without its line. Each command: a period /// run again after its run was reversed (month_reposting), a disposal whose /// month's run was reversed (its final depreciation, month_reposting), a /// daily schedule (schedule_convention), and a schedule month posted again -/// (month_reposting). Taken again, each journals and the books verify. +/// (month_reposting). After the owner's upgrade each journals, and the +/// books verify. #[test] -fn a_refused_journal_feature_journals_nothing() { +fn a_write_waiting_for_the_upgrade_journals_nothing() { let dir = test_dir("feature-refused"); let accounts = schedule_books(&dir); let mut store = Store::open(&dir).unwrap(); @@ -1292,12 +1319,15 @@ fn a_refused_journal_feature_journals_nothing() { store .reverse_entry(jan, day(2026, 1, 31), "doubted") .unwrap(); + // Books as an older version made them (format 1, books()): nothing + // declared, and everything above needs nothing new. assert!(store.journal_format().features.is_empty()); - store.refuse_new_features(true); let head = store.head(); let refused = |r: Result<(), StoreError>, what: &str| match r { - Err(StoreError::FormatRefused(why)) => assert!(why.contains("feature"), "{what}: {why}"), + Err(StoreError::FormatUpgradePending { need }) => { + assert!(need.starts_with("the journal feature "), "{what}: {need}") + } other => panic!("{what}: {other:?}"), }; refused( @@ -1333,7 +1363,7 @@ fn a_refused_journal_feature_journals_nothing() { }); store.verify().unwrap(); - store.refuse_new_features(false); + assert!(store.upgrade_format().unwrap().is_some()); assert!( store .run_depreciation(asset, "2026-01") @@ -1347,9 +1377,6 @@ fn a_refused_journal_feature_journals_nothing() { store .dispose_asset(sold, day(2026, 3, 15), usd(900_000), cash, gain) .unwrap(); - assert_eq!( - store.journal_format().features, - vec![ledger_store::format::MONTH_REPOSTING.to_string()] - ); + assert_eq!(*store.journal_format(), ledger_store::format::current()); store.verify().unwrap(); } diff --git a/crates/ledger-store/tests/format_upgrade.rs b/crates/ledger-store/tests/format_upgrade.rs new file mode 100644 index 0000000..164731f --- /dev/null +++ b/crates/ledger-store/tests/format_upgrade.rs @@ -0,0 +1,390 @@ +//! The format upgrade (plan/served.md 3.9, D-39; rules.md B-27): books an +//! older wanderbooks wrote open on their old journal format, refuse only +//! what needs the new one, and move to this version's format in one +//! journaled step; books this version makes are in its format from their +//! first lines. +//! +//! ```text +//! GOAL: no book is moved past what its owner's older version reads +//! without the owner's go-ahead, and an upgrade changes nothing of the +//! books but what may be written from then on. +//! SOURCE: plan/served.md 3.9 ("Format changes are the server's gate, not +//! the updater's"); D-39; rules.md B-27. +//! INVARIANTS: +//! 1. New books declare this version's format, every feature it knows, +//! as they are created: they wait for no upgrade. +//! 2. Older books open, in the writer and the viewer, without a byte of +//! their journal changing, and say what upgrade they wait for. +//! 3. Before the upgrade a write that needs the new format is refused, +//! journaling nothing; a write that needs nothing new works. +//! 4. The upgrade is one journal line declaring the whole format; asked +//! again, it journals nothing. +//! 5. Every book an earlier version wrote (tests/fixtures/old-books) +//! upgrades, verifies as before and replays to exactly the figures +//! that version computed, in the writer and the viewer. +//! 6. Books declaring a feature this version doesn't know are still +//! refused as newer: the upgrade never weakens the gate. +//! ``` + +use chrono::NaiveDate; +use ledger_core::settings::{Change, Source}; +use ledger_core::{AccountType, Currency, Money, Side}; +use ledger_store::format::{FORMAT_SET, current}; +use ledger_store::{JOURNAL_FILE, JournalFormat, JournalWriter, Snapshot, Store, StoreError}; +use rust_decimal::Decimal; +use serde_json::json; +use std::path::{Path, PathBuf}; + +/// A test's dir, removed when the test ends, pass or panic. +struct TestDir(PathBuf); + +impl std::ops::Deref for TestDir { + type Target = Path; + fn deref(&self) -> &Path { + &self.0 + } +} + +impl Drop for TestDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +fn test_dir(name: &str) -> TestDir { + let dir = std::env::temp_dir().join(format!( + "wanderbooks-format-upgrade-{name}-{}", + std::process::id() + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + TestDir(dir) +} + +fn journal(dir: &Path) -> Vec { + std::fs::read(dir.join(JOURNAL_FILE)).unwrap() +} + +fn lines(dir: &Path) -> Vec { + String::from_utf8(journal(dir)) + .unwrap() + .lines() + .map(|l| serde_json::from_str(l).unwrap()) + .collect() +} + +/// The payload this version's format is declared with. +fn this_versions_declaration() -> serde_json::Value { + let c = current(); + json!({"format": c.format, "features": c.features}) +} + +fn usd(n: i64) -> Money { + Money::new(Decimal::from(n), Currency::USD) +} + +/// Books as an older version made them (journal format 1), with a chart +/// and one posted entry. +fn old_books(dir: &Path) -> Store { + let mut s = Store::create_in_format(dir, Currency::USD, &JournalFormat::default()).unwrap(); + let cash = s + .add_account("1000", "Cash", AccountType::Asset, None, false) + .unwrap(); + let cap = s + .add_account("3000", "Capital", AccountType::Equity, None, false) + .unwrap(); + let e = s + .create_entry(NaiveDate::from_ymd_opt(2026, 3, 1).unwrap(), "Capital in") + .unwrap(); + s.add_line(e, cash, Side::Debit, usd(500)).unwrap(); + s.add_line(e, cap, Side::Credit, usd(500)).unwrap(); + s.post_entry(e).unwrap(); + s +} + +fn a_setting() -> (Vec, Source) { + ( + vec![Change { + key: "company.name".into(), + value: Some(json!("Fernhill Labs LLC")), + }], + Source { + decided_by: "Avery Quill".into(), + decided_on: NaiveDate::from_ymd_opt(2026, 10, 20).unwrap(), + citation: None, + note: None, + }, + ) +} + +/// INVARIANT 1: new books are in this version's format from their second +/// line on, and wait for no upgrade, in the writer and the viewer. +#[test] +fn new_books_are_in_this_versions_format_from_their_creation() { + let dir = test_dir("new"); + let s = Store::create(&dir, Currency::USD).unwrap(); + assert_eq!(s.journal_format(), ¤t()); + assert_eq!(s.journal_format().upgrade(), None); + drop(s); + let l = lines(&dir); + assert_eq!(l.len(), 2, "store.created, then the format"); + assert_eq!(l[0]["type"], "store.created"); + assert_eq!(l[1]["type"], FORMAT_SET); + assert_eq!(l[1]["payload"], this_versions_declaration()); + let s = Store::open(&dir).unwrap(); + assert_eq!(s.journal_format().upgrade(), None); + s.verify().unwrap(); + let snap = Snapshot::load(&dir).unwrap(); + assert_eq!(snap.journal_format(), ¤t()); +} + +/// INVARIANTS 2 to 4: older books open without a byte changing and say +/// what they wait for; a change that needs the new format is refused, +/// journaling nothing, while one that needs nothing new is made; the +/// upgrade is one line, the whole format at once, and asked again does +/// nothing; then the change is made. +#[test] +fn a_change_needing_the_new_format_waits_for_the_upgrade() { + let dir = test_dir("waits"); + drop(old_books(&dir)); + let before = journal(&dir); + let mut s = Store::open(&dir).unwrap(); + assert_eq!(journal(&dir), before, "opening wrote nothing"); + let waiting = s.journal_format().upgrade().expect("they wait"); + assert_eq!(waiting.from, JournalFormat::default()); + assert_eq!(waiting.to, current()); + let snap = Snapshot::load(&dir).unwrap(); + assert_eq!(snap.journal_format().upgrade(), Some(waiting.clone())); + // Refused, saying why, with nothing journaled or applied. + let (changes, source) = a_setting(); + match s.change_settings(&changes, &source) { + Err(e @ StoreError::FormatUpgradePending { .. }) => { + assert!(e.to_string().contains("format_upgrade_pending"), "{e}"); + } + other => panic!("expected FormatUpgradePending, got {other:?}"), + } + assert_eq!(journal(&dir), before); + assert!(!s.ledger().settings().is_journaled()); + for refused in [ + s.set_convention(&ledger_core::conventions::NewConvention { + id: "close-ask".into(), + kind: ledger_core::conventions::ConventionKind::CloseStep, + title: "Ask about cards".into(), + body_md: "Ask.".into(), + structured: None, + source: source.clone(), + }) + .map(drop), + s.raise_open_item(&ledger_core::conventions::NewOpenItem { + title: "Find the lease".into(), + detail_md: None, + due: None, + source: source.clone(), + }) + .map(drop), + ] { + assert!( + matches!(refused, Err(StoreError::FormatUpgradePending { .. })), + "{refused:?}" + ); + } + assert_eq!(journal(&dir), before); + // What needs nothing new is made as ever. + let supplies = s + .add_account("6000", "Supplies", AccountType::Expense, None, false) + .unwrap(); + let cash = s.ledger().chart().find_by_code("1000").unwrap().id; + let e = s + .create_entry(NaiveDate::from_ymd_opt(2026, 3, 2).unwrap(), "Pens") + .unwrap(); + s.add_line(e, supplies, Side::Debit, usd(12)).unwrap(); + s.add_line(e, cash, Side::Credit, usd(12)).unwrap(); + s.post_entry(e).unwrap(); + assert_eq!(s.journal_format(), &JournalFormat::default()); + let figures = ledger_core::reports::snapshot(s.ledger()).unwrap(); + // The upgrade: one line, the whole format. + let mid = journal(&dir); + let n = lines(&dir).len(); + let up = s.upgrade_format().unwrap().expect("upgraded"); + assert_eq!(up, waiting); + assert_eq!(s.journal_format(), ¤t()); + let after = lines(&dir); + assert_eq!(after.len(), n + 1, "one line"); + assert_eq!(after[n]["type"], FORMAT_SET); + assert_eq!(after[n]["payload"], this_versions_declaration()); + assert!(journal(&dir).starts_with(&mid), "nothing before it changed"); + assert_eq!( + ledger_core::reports::snapshot(s.ledger()).unwrap(), + figures, + "the books read as before" + ); + // Again: nothing. + let upgraded = journal(&dir); + assert_eq!(s.upgrade_format().unwrap(), None); + assert_eq!(journal(&dir), upgraded); + // Now the change is made. + s.change_settings(&changes, &source).unwrap(); + assert!(s.ledger().settings().is_journaled()); + s.verify().unwrap(); + drop(s); + let s = Store::open(&dir).unwrap(); + assert_eq!(s.journal_format(), ¤t()); + assert_eq!(s.journal_format().upgrade(), None); + s.verify().unwrap(); + let snap = Snapshot::load(&dir).unwrap(); + assert_eq!(snap.journal_format(), ¤t()); + assert_eq!( + lines(&dir) + .iter() + .filter(|l| l["type"] == FORMAT_SET) + .count(), + 1, + "the format moved exactly once" + ); +} + +/// The books an earlier wanderbooks wrote (tests/fixtures/old-books; see +/// tests/store.rs `old_books`). +fn old_book_fixtures() -> Vec { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/old-books"); + let mut dirs: Vec = std::fs::read_dir(&root) + .unwrap() + .map(|e| e.unwrap().path()) + .filter(|p| p.is_dir()) + .collect(); + dirs.sort(); + assert!(dirs.len() >= 29, "{} old books", dirs.len()); + dirs +} + +/// A fixture's journal and stored files in a fresh dir of their own. +fn fixture_copy(fixture: &Path) -> TestDir { + fn copy(from: &Path, to: &Path) { + std::fs::create_dir_all(to).unwrap(); + for e in std::fs::read_dir(from).unwrap() { + let e = e.unwrap(); + if e.file_name() == "expected.json" || e.file_name() == "record.json" { + continue; + } + if e.file_type().unwrap().is_dir() { + copy(&e.path(), &to.join(e.file_name())); + } else { + std::fs::copy(e.path(), to.join(e.file_name())).unwrap(); + } + } + } + let name = fixture.file_name().unwrap().to_string_lossy(); + let dir = test_dir(&format!("fixture-{name}")); + copy(fixture, &dir); + dir +} + +/// What expected.json records of a ledger (tests/store.rs +/// `old_books_figures`, field for field). +fn figures(l: &ledger_core::Ledger) -> serde_json::Value { + let entries: Vec<_> = (1..l.next_entry_id()) + .filter_map(|id| l.get_entry(id)) + .collect(); + let parties: Vec<_> = (1..l.next_entry_id()) + .map(|id| (id, l.entry_parties(id))) + .filter(|(_, p)| !p.is_empty()) + .collect(); + json!({ + "snapshot": ledger_core::reports::snapshot(l).map_err(|e| e.to_string()), + "entries": entries, + "assets": l.registered_assets().collect::>(), + "periods": l.list_periods(), + "schedules": l.schedules().collect::>(), + "recurring": l.recurring_entries().collect::>(), + "invoices": l.invoices(), + "bills": l.bills(), + "payroll_runs": l.list_payroll_runs(), + "entry_parties": parties, + }) +} + +/// INVARIANTS 2, 4 and 5: every book an earlier version wrote opens +/// without a byte changing, upgrades in one line (or, already in this +/// version's format, waits for nothing), verifies as it did, and replays +/// to exactly the figures that version computed: in the writer, again +/// after reopening, and in the viewer. +#[test] +fn every_book_an_earlier_version_wrote_upgrades_and_still_verifies() { + let mut upgraded = 0; + for fixture in old_book_fixtures() { + let name = fixture.file_name().unwrap().to_string_lossy().to_string(); + let expected: serde_json::Value = + serde_json::from_slice(&std::fs::read(fixture.join("expected.json")).unwrap()).unwrap(); + let dir = fixture_copy(&fixture); + let before = journal(&dir); + let mut s = Store::open(&dir).unwrap_or_else(|e| panic!("{name}: {e}")); + assert_eq!(journal(&dir), before, "{name}: opening wrote"); + let waiting = s.journal_format().upgrade(); + let declared = before + .split(|b| *b == b'\n') + .filter(|l| !l.is_empty()) + .any(|l| serde_json::from_slice::(l).unwrap()["type"] == FORMAT_SET); + if !declared { + assert!(waiting.is_some(), "{name}: format 1 books wait"); + } + let up = s.upgrade_format().unwrap_or_else(|e| panic!("{name}: {e}")); + assert_eq!(up, waiting, "{name}"); + let after = journal(&dir); + assert!(after.starts_with(&before), "{name}: the old lines changed"); + let added = after[before.len()..] + .split(|b| *b == b'\n') + .filter(|l| !l.is_empty()) + .count(); + assert_eq!(added, usize::from(up.is_some()), "{name}: one line"); + if up.is_some() { + upgraded += 1; + } + assert_eq!(s.journal_format(), ¤t(), "{name}"); + let verify = s.verify().map(|_| ()).map_err(|e| e.to_string()); + assert_eq!(json!(verify), expected["verify"], "{name}: verify"); + let mut got = figures(s.ledger()); + got["verify"] = expected["verify"].clone(); + assert_eq!(got, expected, "{name}: the writer's figures"); + drop(s); + let s = Store::open(&dir).unwrap_or_else(|e| panic!("{name}: reopen: {e}")); + assert_eq!(s.journal_format().upgrade(), None, "{name}"); + let mut got = figures(s.ledger()); + got["verify"] = expected["verify"].clone(); + assert_eq!(got, expected, "{name}: reopened"); + drop(s); + let snap = Snapshot::load(&dir).unwrap_or_else(|e| panic!("{name}: viewer: {e}")); + assert_eq!(snap.journal_format(), ¤t(), "{name}"); + let mut got = figures(snap.ledger()); + got["verify"] = expected["verify"].clone(); + assert_eq!(got, expected, "{name}: the viewer's figures"); + } + assert!(upgraded >= 28, "only {upgraded} fixtures were format 1"); +} + +/// INVARIANT 6: books that declare a feature this version doesn't know +/// (a newer version wrote them) are refused as newer, in the writer and +/// the viewer, and never touched: there is nothing to upgrade. +#[test] +fn books_declaring_a_feature_this_version_doesnt_know_stay_refused_as_newer() { + let dir = test_dir("newer"); + drop(old_books(&dir)); + { + let mut w = JournalWriter::open(&dir.join(JOURNAL_FILE)).unwrap(); + w.append( + FORMAT_SET, + json!({"format": 2, "features": ["from_a_newer_version"]}), + ) + .unwrap(); + w.append("account.added", json!({})).unwrap(); + } + let before = journal(&dir); + for refused in [Store::open(&dir).map(drop), Snapshot::load(&dir).map(drop)] { + match refused { + Err(StoreError::NewerJournal { need, .. }) => { + assert_eq!(need, "journal features from_a_newer_version") + } + other => panic!("expected NewerJournal, got {other:?}"), + } + } + assert_eq!(journal(&dir), before); +} diff --git a/crates/ledger-store/tests/member_closes.rs b/crates/ledger-store/tests/member_closes.rs index 6d2bffe..32fefbc 100644 --- a/crates/ledger-store/tests/member_closes.rs +++ b/crates/ledger-store/tests/member_closes.rs @@ -10,7 +10,7 @@ use ledger_core::{ members_equity, }; use ledger_store::journal::read_and_verify_journal; -use ledger_store::{JOURNAL_FILE, JournalWriter, Snapshot, Store, StoreError}; +use ledger_store::{JOURNAL_FILE, JournalFormat, JournalWriter, Snapshot, Store, StoreError}; use rust_decimal::Decimal; use serde_json::{Value, json}; use std::path::{Path, PathBuf}; @@ -82,6 +82,25 @@ fn books(dir: &Path) -> Books { } } +/// [`books`] as a version before the format gate made them: format 1, +/// nothing declared (D-39). +fn old_books(dir: &Path) -> Books { + let mut store = Store::create_in_format(dir, Currency::USD, &JournalFormat::default()).unwrap(); + let mut add = |code: &str, t| store.add_account(code, code, t, None, false).unwrap(); + let cash = add("1000", AccountType::Asset); + let caps = [ + add("3000", AccountType::Equity), + add("3001", AccountType::Equity), + ]; + let sales = add("4000", AccountType::Revenue); + Books { + store, + cash, + caps, + sales, + } +} + /// [`books`] opened again. fn reopen(dir: &Path) -> Books { let store = Store::open(dir).unwrap(); @@ -147,11 +166,14 @@ fn three_cumulative_closes(dir: &Path) -> Books { b } -/// Contract (rules.md B-27; P3 defect 17): the first members close counted -/// from its regime's start declares the `members_cumulative` feature, on -/// the line just before it and once only, so an older version stops at the -/// declaration (it would replay the close under the run rule). Books that -/// never close to the members never declare it. +/// Contract (rules.md B-27; P3 defect 17; D-39): a members close counted +/// from its regime's start needs the `members_cumulative` feature, so an +/// older version stops at its declaration (it would replay the close under +/// the run rule). New books declare it on their second line, with the rest +/// of this version's format, and never again; books an older version made +/// (format 1) refuse a members close, journaling nothing, until the +/// owner's format upgrade declares it, and close to retained earnings as +/// ever. #[test] fn the_first_cumulative_close_declares_its_feature() { let dir = test_dir("declares"); @@ -169,12 +191,13 @@ fn the_first_cumulative_close_declares_its_feature() { .filter(|(_, (t, p))| t == "journal.format_set" && declares(p)) .map(|(i, _)| i) .collect(); - let first_close = lines - .iter() - .position(|(t, p)| t == "period.closed" && p["close_to"]["rounding"] == "cumulative_since") - .unwrap(); - assert_eq!(declarations, [first_close - 1]); - assert_eq!(lines[first_close - 1].1["format"], 2); + // Line 2: this version's whole format, as the books were created. + assert_eq!(declarations, [1]); + assert_eq!(lines[1].1["format"], 2); + assert_eq!( + lines[1].1["features"], + json!(ledger_store::format::current().features) + ); assert_eq!( lines .iter() @@ -193,17 +216,47 @@ fn the_first_cumulative_close_declares_its_feature() { .count(); assert_eq!(again, 1); - // Books that close only to retained earnings never declare it. - let dir = test_dir("declares-not"); - let mut b = books(&dir); + // Books an older version made: a members close waits for the upgrade, + // a close to retained earnings doesn't. + let dir = test_dir("declares-old"); + let mut b = old_books(&dir); earn_and_close( &mut b, 1, "0.01", &CloseTo::RetainedEarnings { sweep: vec![] }, ); - drop(b); + let e = b.store.create_entry(d(2, 10), "sale").unwrap(); + let money = Money::new(dec("0.01"), Currency::USD); + b.store.add_line(e, b.cash, Side::Debit, money).unwrap(); + b.store.add_line(e, b.sales, Side::Credit, money).unwrap(); + b.store.post_entry(e).unwrap(); + let p = b + .store + .create_period("2026-02", d(2, 1), month_end(2)) + .unwrap(); + let head = b.store.head(); + let to = CloseTo::Members(half(&b.caps)); + match b.store.close_period_to(p, &to) { + Err(StoreError::FormatUpgradePending { need }) => { + assert_eq!(need, "the journal feature members_cumulative") + } + other => panic!("expected FormatUpgradePending, got {other:?}"), + } + assert_eq!(b.store.head(), head, "refused whole"); assert!(!journal(&dir).iter().any(|(_, p)| declares(p))); + // The owner's go-ahead: the whole format at once, then the close. + assert!(b.store.upgrade_format().unwrap().is_some()); + b.store.close_period_to(p, &to).unwrap().unwrap(); + drop(b); + let lines = journal(&dir); + let n = lines.len(); + assert_eq!(lines[n - 1].0, "period.closed"); + assert_eq!(lines[n - 2].0, "journal.format_set"); + assert_eq!( + lines[n - 2].1["features"], + json!(ledger_store::format::current().features) + ); } /// A change to a journaled close's target. @@ -457,6 +510,18 @@ fn the_first_close_after_the_upgrade_trues_up_the_old_rules() { distributions: None, }, ]; + // Books an earlier version wrote: the close counted from its regime's + // start waits for the owner's go-ahead (D-39), journaling nothing; the + // upgrade declares this version's whole format. + let head = store.head(); + match store.close_period_to(dec_period, &CloseTo::Members(members.clone())) { + Err(StoreError::FormatUpgradePending { need }) => { + assert_eq!(need, "the journal feature members_cumulative") + } + other => panic!("expected FormatUpgradePending, got {other:?}"), + } + assert_eq!(store.head(), head, "refused whole"); + assert!(store.upgrade_format().unwrap().is_some()); let closing = store .close_period_to(dec_period, &CloseTo::Members(members)) .unwrap() @@ -504,14 +569,18 @@ fn the_first_close_after_the_upgrade_trues_up_the_old_rules() { capital_on(store.ledger(), d(12, 31)), [dec("99.08"), dec("100.07")] ); - // One declaration, just before the close. + // One declaration, the upgrade's, just before the close: this + // version's whole format. drop(store); let lines = journal(&dir); let n = lines.len(); assert_eq!(lines[n - 1].0, "period.closed"); assert_eq!(lines[n - 1].1["close_to"]["since_seq"], 1); assert_eq!(lines[n - 2].0, "journal.format_set"); - assert_eq!(lines[n - 2].1["features"], json!(["members_cumulative"])); + assert_eq!( + lines[n - 2].1["features"], + json!(ledger_store::format::current().features) + ); // Reopened, and in the viewer: the same. let store = Store::open(&dir).unwrap(); store.verify().unwrap(); diff --git a/crates/ledger-store/tests/parties.rs b/crates/ledger-store/tests/parties.rs index a907fc4..73df9ed 100644 --- a/crates/ledger-store/tests/parties.rs +++ b/crates/ledger-store/tests/parties.rs @@ -16,7 +16,7 @@ use ledger_core::{ PayrollLine, PayrollRun, Role, Side, W4Facts, W9Facts, }; use ledger_store::journal::read_and_verify_journal; -use ledger_store::{DB_FILE, JOURNAL_FILE, JournalWriter, Store}; +use ledger_store::{JOURNAL_FILE, JournalWriter, Store, db_file}; use rust_decimal::Decimal; use serde_json::{Value, json}; use std::path::{Path, PathBuf}; @@ -192,7 +192,7 @@ fn view(l: &Ledger) -> View { fn delete_db(dir: &Path) { for suffix in ["", "-wal", "-shm", "-journal"] { - let _ = std::fs::remove_file(dir.join(format!("{DB_FILE}{suffix}"))); + let _ = std::fs::remove_file(dir.join(format!("{}{suffix}", db_file()))); } } diff --git a/crates/ledger-store/tests/party_creditors.rs b/crates/ledger-store/tests/party_creditors.rs index 1eebc1a..148dc33 100644 --- a/crates/ledger-store/tests/party_creditors.rs +++ b/crates/ledger-store/tests/party_creditors.rs @@ -1,18 +1,22 @@ //! A party named the creditor of a liability (rules.md B-35, P3 defect //! 24), in the store: journaled as `party.creditor_set {id, account_id, -//! on}`, after the journal declares the feature it needs -//! (`party_creditors`) the first time, and never otherwise. +//! on}`, which needs the journal feature `party_creditors`: books this +//! version made declared it when they were created; books an older +//! version wrote refuse it until their format upgrade (rules.md B-27). //! //! Oracles: `Store::set_party_creditor`'s doc, `format::KNOWN_FEATURES` -//! and `PARTY_CREDITORS`, `Store::ensure_feature`'s doc (nothing journaled -//! when the feature is in force; books that never use it never change), -//! and `Ledger::set_party_creditor`'s SCENARIOS. Expected values by hand. +//! and `PARTY_CREDITORS`, `Store::ensure_feature`'s doc (refused, +//! journaling nothing, unless the books declared it; books that never use +//! it never change), and `Ledger::set_party_creditor`'s SCENARIOS. +//! Expected values by hand. use chrono::NaiveDate; use ledger_core::{AccountType, Currency, Money, Side}; -use ledger_store::format::{FORMAT_SET, KNOWN_FEATURES, PARTY_CREDITORS}; +use ledger_store::format::{FORMAT_SET, KNOWN_FEATURES, PARTY_CREDITORS, current}; use ledger_store::journal::read_and_verify_journal; -use ledger_store::{DB_FILE, JOURNAL_FILE, JournalWriter, Snapshot, Store}; +use ledger_store::{ + JOURNAL_FILE, JournalFormat, JournalWriter, Snapshot, Store, StoreError, db_file, +}; use rust_decimal::Decimal; use serde_json::{Value, json}; use std::path::{Path, PathBuf}; @@ -55,7 +59,13 @@ fn bytes(dir: &Path) -> Vec { /// Books with a card, the members' financing account, a member, and a /// posted entry. fn books(dir: &Path) -> (Store, u64, u64, u64, u64) { - let mut s = Store::create(dir, Currency::USD).unwrap(); + books_in(dir, ¤t()) +} + +/// [`books`] in the journal format `format`: format 1 +/// ([`JournalFormat::default`]) is books an older version wrote. +fn books_in(dir: &Path, format: &JournalFormat) -> (Store, u64, u64, u64, u64) { + let mut s = Store::create_in_format(dir, Currency::USD, format).unwrap(); let cash = s .add_account("1000", "Checking", AccountType::Asset, None, false) .unwrap(); @@ -103,40 +113,48 @@ fn party_creditors_is_a_known_feature() { assert!(KNOWN_FEATURES.contains(&PARTY_CREDITORS)); } -/// Store::set_party_creditor doc: the first one declares the feature (one -/// journal.format_set keeping the format, at least 2, and every feature -/// already declared), then journals party.creditor_set; later ones journal -/// one line each; the books replay to the same directory, through the -/// store and the viewer's snapshot alike. +/// Store::set_party_creditor doc: books this version made declared the +/// feature when they were created (their second line, the only +/// declaration), so each creditor named journals one line, +/// party.creditor_set, and nothing else; the books replay to the same +/// directory, through the store and the viewer's snapshot alike. #[test] -fn naming_a_creditor_declares_its_feature_once() { +fn naming_a_creditor_journals_one_line_each_time() { let dir = test_dir("declare"); let (mut s, card, due, issuer, tobin) = books(&dir); - let before = journal(&dir).len(); + let j = journal(&dir); + assert_eq!(j[1].0, FORMAT_SET); + assert!( + j[1].1["features"] + .as_array() + .unwrap() + .contains(&json!("party_creditors")) + ); + let before = j.len(); s.set_party_creditor(issuer, card, true).unwrap(); let j = journal(&dir); - assert_eq!(j.len(), before + 2); - assert_eq!(j[before].0, FORMAT_SET); + assert_eq!(j.len(), before + 1); assert_eq!( - j[before].1, - json!({"format": 2, "features": ["party_creditors"]}) - ); - assert_eq!( - j[before + 1], + j[before], ( "party.creditor_set".to_string(), json!({"id": issuer, "account_id": card, "on": true}) ) ); - assert_eq!(s.journal_format().features, vec!["party_creditors"]); s.set_party_creditor(tobin, due, true).unwrap(); s.set_party_creditor(tobin, due, false).unwrap(); let j = journal(&dir); - assert_eq!(j.len(), before + 4, "the feature is declared once"); + assert_eq!(j.len(), before + 3, "one line each"); assert_eq!( - j[before + 3].1, + j[before + 2].1, json!({"id": tobin, "account_id": due, "on": false}) ); + assert_eq!( + j.iter().filter(|(t, _)| t == FORMAT_SET).count(), + 1, + "nothing declared on the way" + ); + assert_eq!(s.journal_format(), ¤t()); s.set_party_creditor(tobin, due, true).unwrap(); let live = s.ledger().clone(); s.verify().unwrap(); @@ -148,15 +166,40 @@ fn naming_a_creditor_declares_its_feature_once() { drop(s); let snap = Snapshot::load(&dir).unwrap(); assert!(*snap.ledger() == live); - assert_eq!(snap.journal_format().features, vec!["party_creditors"]); + assert_eq!(snap.journal_format(), ¤t()); // Rebuilt from the journal alone. for suffix in ["", "-wal", "-shm"] { - let _ = std::fs::remove_file(dir.join(format!("{DB_FILE}{suffix}"))); + let _ = std::fs::remove_file(dir.join(format!("{}{suffix}", db_file()))); } let s = Store::open(&dir).unwrap(); assert!(*s.ledger() == live); } +/// Books an older version wrote (format 1, declaring nothing) refuse a +/// creditor while they wait for their format upgrade, journaling nothing, +/// not the line and not the feature; once the owner upgrades them, the +/// same command journals its one line. +#[test] +fn a_creditor_in_older_books_waits_for_the_format_upgrade() { + let dir = test_dir("older"); + let (mut s, card, _, issuer, _) = books_in(&dir, &JournalFormat::default()); + let was = bytes(&dir); + assert!(matches!( + s.set_party_creditor(issuer, card, true), + Err(StoreError::FormatUpgradePending { need }) if need.contains("party_creditors") + )); + assert_eq!(bytes(&dir), was, "nothing journaled"); + assert!(s.ledger().creditors_of(card).is_empty()); + assert!(s.upgrade_format().unwrap().is_some()); + let before = journal(&dir).len(); + s.set_party_creditor(issuer, card, true).unwrap(); + let j = journal(&dir); + assert_eq!(j.len(), before + 1); + assert_eq!(j[before].0, "party.creditor_set"); + s.verify().unwrap(); + assert_eq!(s.ledger().creditors_of(card), vec![issuer]); +} + /// A refusal journals nothing: not the line, and not the feature either. #[test] fn a_refused_creditor_journals_nothing() { @@ -164,34 +207,42 @@ fn a_refused_creditor_journals_nothing() { let (mut s, _, _, issuer, _) = books(&dir); let cash = s.ledger().chart().find_by_code("1000").unwrap().id; let was = bytes(&dir); + let format = s.journal_format().clone(); for (party, account) in [(issuer, cash), (999, cash), (issuer, 999)] { assert!(s.set_party_creditor(party, account, true).is_err()); assert_eq!(bytes(&dir), was); } - assert!(s.journal_format().features.is_empty()); + assert_eq!(s.journal_format(), &format); } -/// Books that never name a creditor never declare the feature: opening, -/// verifying and rebuilding them leaves the journal byte-identical, in the -/// format it was in. +/// Books that never name a creditor are never changed by the feature: +/// opening, verifying and rebuilding them leaves the journal +/// byte-identical, in the format it was in, this version's or an older +/// one's (which declares nothing and stays so). #[test] fn books_without_creditors_are_never_changed_by_the_feature() { - let dir = test_dir("untouched"); - let (s, ..) = books(&dir); - let format = s.journal_format().clone(); - drop(s); - let was = bytes(&dir); - let mut s = Store::open(&dir).unwrap(); - s.verify().unwrap(); - s.rebuild().unwrap(); - assert_eq!(s.journal_format(), &format); - drop(s); - assert_eq!(bytes(&dir), was); - assert!( - journal(&dir) - .iter() - .all(|(t, p)| t != FORMAT_SET || p["features"] == json!([])) - ); + for (name, format) in [ + ("untouched", current()), + ("untouched-old", JournalFormat::default()), + ] { + let dir = test_dir(name); + let (s, ..) = books_in(&dir, &format); + assert_eq!(s.journal_format(), &format); + drop(s); + let was = bytes(&dir); + let mut s = Store::open(&dir).unwrap(); + s.verify().unwrap(); + s.rebuild().unwrap(); + assert_eq!(s.journal_format(), &format, "{name}"); + drop(s); + assert_eq!(bytes(&dir), was, "{name}"); + let j = journal(&dir); + assert!(j.iter().all(|(t, _)| t != "party.creditor_set"), "{name}"); + assert!( + j.iter().filter(|(t, _)| t == FORMAT_SET).count() <= 1, + "{name}: nothing declared but at creation" + ); + } } /// A line that can't replay is refused on open, with the database and @@ -224,7 +275,7 @@ fn corrupt_creditor_lines_are_refused_on_open() { drop(w); assert!(Store::open(&dir).is_err(), "{name}: {payload} opened"); for suffix in ["", "-wal", "-shm"] { - let _ = std::fs::remove_file(dir.join(format!("{DB_FILE}{suffix}"))); + let _ = std::fs::remove_file(dir.join(format!("{}{suffix}", db_file()))); } assert!( Store::open(&dir).is_err(), diff --git a/crates/ledger-store/tests/party_tin.rs b/crates/ledger-store/tests/party_tin.rs index 375a35a..16ce67e 100644 --- a/crates/ledger-store/tests/party_tin.rs +++ b/crates/ledger-store/tests/party_tin.rs @@ -8,7 +8,7 @@ use chrono::NaiveDate; use ledger_core::{Currency, Role, TinFacts, TinType, W9Facts}; use ledger_store::journal::read_and_verify_journal; -use ledger_store::{DB_FILE, JOURNAL_FILE, JournalWriter, Store}; +use ledger_store::{JOURNAL_FILE, JournalWriter, Store, db_file}; use serde_json::{Value, json}; use std::path::{Path, PathBuf}; @@ -72,7 +72,7 @@ fn w9(last4: &str) -> W9Facts { fn delete_db(dir: &Path) { for suffix in ["", "-wal", "-shm", "-journal"] { - let _ = std::fs::remove_file(dir.join(format!("{DB_FILE}{suffix}"))); + let _ = std::fs::remove_file(dir.join(format!("{}{suffix}", db_file()))); } } diff --git a/crates/ledger-store/tests/root.rs b/crates/ledger-store/tests/root.rs index f33110d..6654565 100644 --- a/crates/ledger-store/tests/root.rs +++ b/crates/ledger-store/tests/root.rs @@ -45,7 +45,14 @@ fn set(key: &str, value: serde_json::Value) -> Change { /// Books with accounts, a posted entry with its receipt, a filed document, /// and (with `settings`) settings of their own: what adoption must carry. fn source_books(dir: &Path, settings: bool) -> Store { - let mut s = Store::create(dir, Currency::USD).unwrap(); + // As an earlier version wrote them: journal format 1, or 2 once they + // keep their own settings (this version's own books declare their + // format as they are made). + let format = ledger_store::JournalFormat { + format: if settings { 2 } else { 1 }, + features: Vec::new(), + }; + let mut s = Store::create_in_format(dir, Currency::USD, &format).unwrap(); if settings { s.adopt_settings(&[set("company.name", json!("Adopted LLC"))], &src("Owner")) .unwrap(); @@ -299,6 +306,9 @@ fn adopted_books_are_the_same_books_and_the_source_is_untouched() { let source = source_books(&data, true); let source_head = source.head(); let source_ledger = source.ledger().clone(); + // Format 2 with no feature, as P2 wrote them: this version knows + // features they haven't declared, so they wait for the upgrade. + assert!(source.journal_format().upgrade().is_some()); drop(source); // A vault and a key beside them, as the old layout kept them. let vault = old.path().join("vault"); @@ -322,9 +332,14 @@ fn adopted_books_are_the_same_books_and_the_source_is_untouched() { ) .unwrap(); assert_eq!(report.source_head, source_head); + // Books with settings gain no event but the copy's move to this + // version's format (new books of this version; the source keeps its + // own): one journal.format_set. + assert!(report.format_upgraded); assert_eq!( - report.head, source_head, - "books with settings gain no event" + report.head.0, + source_head.0 + 1, + "books with settings gain only the format" ); assert_eq!(report.blob_files, 2); assert_eq!(report.blobs_verified, 2); @@ -405,6 +420,9 @@ fn books_without_settings_need_them_and_get_them_on_the_way_in() { .unwrap(); assert_eq!(report.source_head, head); assert_eq!(report.settings_imported, 1); + // The copy is new books of this version: in its journal format (the + // source, format 1, keeps its own). + assert!(report.format_upgraded); // What was added: the journal format the settings need, declared // first (older books), then the settings. let journal = std::fs::read_to_string(root.book_dir(&id("acme")).join(JOURNAL_FILE)).unwrap(); @@ -437,6 +455,7 @@ fn books_without_settings_need_them_and_get_them_on_the_way_in() { assert_eq!(added.last().map(String::as_str), Some("settings.set")); assert_eq!(report.head.0, head.0 + added.len() as u64); let adopted = Store::open(&root.book_dir(&id("acme"))).unwrap(); + assert_eq!(adopted.journal_format(), &ledger_store::format::current()); assert!(adopted.ledger().settings().is_journaled()); assert_eq!( adopted.ledger().settings().company("USD").name, @@ -881,6 +900,9 @@ fn settings_given_for_books_that_keep_their_own_are_reported_unused() { let old = tmp("adopt-ignored-src"); let s = source_books(old.path(), true); let head = s.head(); + // Format 2 already: moved to this version's only if it knows features + // these books haven't declared. + let waits = s.journal_format().upgrade().is_some(); drop(s); let report = root .adopt( @@ -893,7 +915,14 @@ fn settings_given_for_books_that_keep_their_own_are_reported_unused() { ) .unwrap(); assert!(report.settings_ignored); - assert_eq!((report.head, report.settings_imported), (head, 0)); + assert_eq!(report.format_upgraded, waits); + assert_eq!( + (report.head.0, report.settings_imported), + (head.0 + u64::from(waits), 0) + ); + if !waits { + assert_eq!(report.head, head); + } } #[test] diff --git a/crates/ledger-store/tests/schema_files.rs b/crates/ledger-store/tests/schema_files.rs new file mode 100644 index 0000000..73b403a --- /dev/null +++ b/crates/ledger-store/tests/schema_files.rs @@ -0,0 +1,371 @@ +//! Each projection schema in a file of its own (ledger_store::db_file_for; +//! plan/served.md 3.9: before the owner consents to a format upgrade, the +//! way back to an older version is always there; rules.md B-27). +//! +//! ```text +//! GOAL: an older version, gone back to, finds the books it can open, +//! however far a newer one moved the projection. +//! INVARIANTS: +//! 1. This version opens or creates only its own schema's file (the +//! legacy `wanderbooks.db` only when that already holds its schema); +//! its own missing, it is built from the journal. +//! 2. A file of any other schema, older or newer, is never migrated, +//! written, refused or deleted: it is left byte for byte, and still +//! whole for its own schema. +//! 3. A damaged file of this version's own schema is kept aside and +//! rebuilt, as ever. +//! 4. Which file is in use, and which others are there, is said. +//! ``` +//! +//! The P2-shaped fixture (tests/fixtures/p2-books) is a books dir as P2 +//! left it: a format 1 journal and its projection in `wanderbooks.db`, at +//! schema 12. While this version's schema is 12 too, it is used as it is; +//! once a schema of P3's moves past it, it must be left byte for byte. A +//! legacy file one schema older than this version's is made here too, so +//! the second case runs on every version. + +use chrono::NaiveDate; +use ledger_core::{AccountType, Currency, Money, Side}; +use ledger_store::schema::{MIGRATIONS, SCHEMA_VERSION}; +use ledger_store::{ + DiskCheck, JOURNAL_FILE, JournalFormat, LEGACY_DB_FILE, Snapshot, Store, db_file, db_file_for, +}; +use rust_decimal::Decimal; +use std::path::{Path, PathBuf}; + +/// The schema P2 kept in `wanderbooks.db`, the last before each schema had +/// a file of its own. +const P2_SCHEMA: u32 = 12; + +/// A test's dir, removed when the test ends, pass or panic. +struct TestDir(PathBuf); + +impl std::ops::Deref for TestDir { + type Target = Path; + fn deref(&self) -> &Path { + &self.0 + } +} + +impl Drop for TestDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +fn test_dir(name: &str) -> TestDir { + let dir = std::env::temp_dir().join(format!( + "wanderbooks-schema-files-{name}-{}", + std::process::id() + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + TestDir(dir) +} + +fn usd(n: i64) -> Money { + Money::new(Decimal::from(n), Currency::USD) +} + +/// Books with a chart and one posted entry (cash 500), closed again, in +/// journal format `format`. +fn books(dir: &Path, format: &JournalFormat) { + let mut s = Store::create_in_format(dir, Currency::USD, format).unwrap(); + let cash = s + .add_account("1000", "Cash", AccountType::Asset, None, false) + .unwrap(); + let cap = s + .add_account("3000", "Capital", AccountType::Equity, None, false) + .unwrap(); + let e = s + .create_entry(NaiveDate::from_ymd_opt(2026, 3, 1).unwrap(), "Capital in") + .unwrap(); + s.add_line(e, cash, Side::Debit, usd(500)).unwrap(); + s.add_line(e, cap, Side::Credit, usd(500)).unwrap(); + s.post_entry(e).unwrap(); +} + +/// The cash balance the books say. +fn cash(s: &Store) -> Decimal { + let id = s.ledger().chart().find_by_code("1000").unwrap().id; + s.ledger().account_balance(id).unwrap() +} + +/// A file's bytes, or None. +fn bytes(path: &Path) -> Option> { + std::fs::read(path).ok() +} + +/// What a projection file says of itself, read without writing it: its +/// schema, the journal lines it applied, and SQLite's integrity check. +fn holds(path: &Path) -> (u32, u64, String) { + let uri = format!("file:{}?mode=ro&immutable=1", path.display()); + let conn = rusqlite::Connection::open_with_flags( + uri, + rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY | rusqlite::OpenFlags::SQLITE_OPEN_URI, + ) + .unwrap(); + let meta = |k: &str| -> String { + conn.query_row("SELECT value FROM meta WHERE key=?1", [k], |r| r.get(0)) + .unwrap() + }; + let check: String = conn + .query_row("PRAGMA integrity_check", [], |r| r.get(0)) + .unwrap(); + ( + meta("schema_version").parse().unwrap(), + meta("last_applied_seq").parse().unwrap(), + check, + ) +} + +/// A copy of tests/fixtures/p2-books. +fn p2_books() -> TestDir { + let from = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/p2-books"); + let dir = test_dir("p2"); + for name in [JOURNAL_FILE, LEGACY_DB_FILE] { + std::fs::copy(from.join(name), dir.join(name)).unwrap(); + } + dir +} + +/// How tests/fixtures/p2-books was written: by this code while its schema +/// was P2's (12, and every line applied as P2 applies it), its projection +/// then given P2's file name. Run with WB_WRITE_P2_FIXTURE=DIR. +#[test] +fn write_the_p2_fixture() { + let Ok(out) = std::env::var("WB_WRITE_P2_FIXTURE") else { + return; + }; + assert_eq!( + SCHEMA_VERSION, P2_SCHEMA, + "only a P2-schema version writes it" + ); + let out = PathBuf::from(out); + books(&out, &JournalFormat::default()); + std::fs::rename(out.join(db_file()), out.join(LEGACY_DB_FILE)).unwrap(); + let _ = std::fs::remove_file(out.join(ledger_store::LOCK_FILE)); + let _ = std::fs::remove_dir(out.join("blobs")); +} + +/// INVARIANTS 1, 2 and 4, on a books dir exactly as P2 left it: with this +/// version's schema P2's, its file is this version's and is used as it is +/// (no rebuild); past it, the file is left byte for byte and whole for its +/// schema, and this version builds its own from the journal. Either way the +/// books open to the same figures and verify. +#[test] +fn a_p2_books_dirs_database_is_used_at_its_schema_and_left_alone_past_it() { + let dir = p2_books(); + let legacy = dir.join(LEGACY_DB_FILE); + let before = bytes(&legacy).unwrap(); + let lines = std::fs::read_to_string(dir.join(JOURNAL_FILE)) + .unwrap() + .lines() + .count() as u64; + assert_eq!(holds(&legacy), (P2_SCHEMA, lines, "ok".into())); + let s = Store::open(&dir).unwrap(); + assert_eq!(cash(&s), Decimal::from(500)); + s.verify().unwrap(); + if SCHEMA_VERSION == P2_SCHEMA { + // Its schema: used as it is, nothing rebuilt or made beside it. + assert_eq!(s.database_file(), LEGACY_DB_FILE); + assert!(s.other_databases().is_empty()); + drop(s); + assert!(!dir.join(db_file()).exists(), "no rebuild for nothing"); + } else { + assert_eq!(s.database_file(), db_file()); + assert_eq!(s.other_databases(), [LEGACY_DB_FILE]); + drop(s); + assert_eq!(bytes(&legacy).unwrap(), before, "P2's file, byte for byte"); + assert_eq!(holds(&legacy), (P2_SCHEMA, lines, "ok".into()), "and whole"); + let s = Store::open(&dir).unwrap(); + assert_eq!(cash(&s), Decimal::from(500)); + drop(s); + assert_eq!(bytes(&legacy).unwrap(), before); + } +} + +/// INVARIANTS 1, 2 and 4, on every version: a legacy file one schema older +/// than this version's (as P2's is to P3's) is never migrated or written. +/// It stays byte for byte, and whole for its own schema; this version's +/// file is built from the journal and caught up as the books move on; the +/// report names the one in use and the one left alone; the viewer checks +/// the one in use. +#[test] +fn an_older_schemas_file_is_left_byte_for_byte() { + let dir = test_dir("older"); + books(&dir, &JournalFormat::default()); + let own = dir.join(db_file()); + let legacy = dir.join(LEGACY_DB_FILE); + // The older version's file: its schema, its own copy of the books. + let older = SCHEMA_VERSION - 1; + { + let conn = rusqlite::Connection::open(&legacy).unwrap(); + for m in &MIGRATIONS[..older as usize] { + conn.execute_batch(m).unwrap(); + } + for (k, v) in [ + ("schema_version", older.to_string()), + ("base_currency", "USD".into()), + ("last_applied_seq", "0".into()), + ] { + conn.execute( + "INSERT OR REPLACE INTO meta(key, value) VALUES (?1, ?2)", + [k, v.as_str()], + ) + .unwrap(); + } + } + std::fs::remove_file(&own).unwrap(); + let before = bytes(&legacy).unwrap(); + let mut s = Store::open(&dir).unwrap(); + assert_eq!(s.database_file(), db_file()); + assert_eq!(s.other_databases(), [LEGACY_DB_FILE]); + assert_eq!(cash(&s), Decimal::from(500)); + let report = s.verify_and_heal().unwrap(); + assert_eq!(report.healed, None); + assert_eq!(report.database, db_file()); + assert_eq!(report.other_databases, [LEGACY_DB_FILE]); + // The books move on: this version's file follows, the other doesn't. + s.add_account("6000", "Fees", AccountType::Expense, None, false) + .unwrap(); + let head = s.head().0; + drop(s); + assert_eq!(bytes(&legacy).unwrap(), before, "never migrated or written"); + assert!(!dir.join(format!("{LEGACY_DB_FILE}-wal")).exists()); + assert_eq!( + holds(&legacy), + (older, 0, "ok".into()), + "whole, at its schema" + ); + assert_eq!(holds(&own), (SCHEMA_VERSION, head, "ok".into())); + let snap = Snapshot::load(&dir).unwrap(); + assert_eq!(snap.check_disk(), DiskCheck::Matches); + let s = Store::open(&dir).unwrap(); + s.verify().unwrap(); + drop(s); + assert_eq!(bytes(&legacy).unwrap(), before); +} + +/// INVARIANT 2: a newer schema's file, whether under its own name or the +/// legacy one, is left alone, never refused: the books open on this +/// version's own file (built from the journal if it isn't there). +#[test] +fn a_newer_schemas_file_is_left_alone_not_refused() { + let dir = test_dir("newer"); + books(&dir, &JournalFormat::default()); + let own = dir.join(db_file()); + // A newer version's files: its own name, and (as no version writes, but + // one could find) the legacy name holding a newer schema. + let newer = dir.join(db_file_for(SCHEMA_VERSION + 1)); + let legacy = dir.join(LEGACY_DB_FILE); + for f in [&newer, &legacy] { + std::fs::copy(&own, f).unwrap(); + let conn = rusqlite::Connection::open(f).unwrap(); + conn.execute( + "UPDATE meta SET value=?1 WHERE key='schema_version'", + [(SCHEMA_VERSION + 1).to_string()], + ) + .unwrap(); + conn.execute_batch("PRAGMA journal_mode=DELETE").unwrap(); + } + let (newer_bytes, legacy_bytes) = (bytes(&newer).unwrap(), bytes(&legacy).unwrap()); + // This version's own file there, then not. + for keep_own in [true, false] { + if !keep_own { + std::fs::remove_file(&own).unwrap(); + } + let s = Store::open(&dir).unwrap(); + assert_eq!(s.database_file(), db_file(), "{keep_own}"); + let mut others = vec![LEGACY_DB_FILE.to_string(), db_file_for(SCHEMA_VERSION + 1)]; + others.sort(); + assert_eq!(s.other_databases(), others); + assert_eq!(cash(&s), Decimal::from(500)); + s.verify().unwrap(); + drop(s); + assert_eq!(bytes(&newer).unwrap(), newer_bytes, "{keep_own}"); + assert_eq!(bytes(&legacy).unwrap(), legacy_bytes, "{keep_own}"); + } +} + +/// INVARIANT 3: this version's own file, damaged (not a database, or +/// holding another schema though named for this one), is kept aside and +/// rebuilt from the journal; a damaged legacy file is another version's to +/// mend, and left as it is. +#[test] +fn a_damaged_file_of_this_versions_schema_is_rebuilt() { + let dir = test_dir("damaged"); + books(&dir, &JournalFormat::default()); + let own = dir.join(db_file()); + let kept_aside = |dir: &Path| { + std::fs::read_dir(dir) + .unwrap() + .filter_map(|e| e.ok()) + .filter(|e| e.file_name().to_string_lossy().contains(".corrupt.")) + .count() + }; + // Not a database at all. + std::fs::write(&own, b"this is not a sqlite database at all").unwrap(); + for wal in ["-wal", "-shm"] { + let _ = std::fs::remove_file(dir.join(format!("{}{wal}", db_file()))); + } + let s = Store::open(&dir).unwrap(); + assert_eq!(cash(&s), Decimal::from(500)); + s.verify().unwrap(); + drop(s); + assert_eq!(kept_aside(&dir), 1); + assert_eq!(holds(&own).0, SCHEMA_VERSION); + // Named for this schema, holding another. + { + let conn = rusqlite::Connection::open(&own).unwrap(); + conn.execute( + "UPDATE meta SET value=?1 WHERE key='schema_version'", + [(SCHEMA_VERSION + 1).to_string()], + ) + .unwrap(); + } + std::thread::sleep(std::time::Duration::from_millis(1100)); + let s = Store::open(&dir).unwrap(); + s.verify().unwrap(); + drop(s); + assert_eq!(kept_aside(&dir), 2); + assert_eq!(holds(&own).0, SCHEMA_VERSION); + // A damaged legacy file: not this version's to mend. + let legacy = dir.join(LEGACY_DB_FILE); + std::fs::write(&legacy, b"garbage of an older version's").unwrap(); + let s = Store::open(&dir).unwrap(); + assert_eq!(s.database_file(), db_file()); + s.verify().unwrap(); + drop(s); + assert_eq!( + bytes(&legacy).unwrap(), + b"garbage of an older version's".to_vec() + ); + assert_eq!(kept_aside(&dir), 2); +} + +/// New books get this version's own schema's file, named for its schema, +/// and nothing else. +#[test] +fn new_books_get_their_schemas_own_file() { + let dir = test_dir("new"); + let s = Store::create(&dir, Currency::USD).unwrap(); + assert_eq!(s.database_file(), db_file()); + assert_eq!(db_file(), format!("wanderbooks.v{SCHEMA_VERSION}.db")); + assert!(s.other_databases().is_empty()); + drop(s); + assert!(dir.join(db_file()).exists()); + assert!(!dir.join(LEGACY_DB_FILE).exists()); + for (name, is) in [ + ("wanderbooks.db", true), + ("wanderbooks.v13.db", true), + ("wanderbooks.v1.db", true), + ("wanderbooks.v.db", false), + ("wanderbooks.v13.db-wal", false), + ("wanderbooks.v13a.db", false), + ("wanderbooks.v13.corrupt.20261021T000000Z", false), + ("journal.jsonl", false), + ] { + assert_eq!(ledger_store::is_database_file(name), is, "{name}"); + } +} diff --git a/crates/ledger-store/tests/settings.rs b/crates/ledger-store/tests/settings.rs index c4f9c64..d7868c0 100644 --- a/crates/ledger-store/tests/settings.rs +++ b/crates/ledger-store/tests/settings.rs @@ -110,23 +110,26 @@ fn assert_every_reader_agrees(dir: &std::path::Path, s: &Store) { assert!(snap.ledger().differs_from(s.ledger()).is_empty()); } -/// The first change declares journal format 2 before it (format gate), and -/// only once; a change to what is already set journals nothing. +/// Settings need journal format 2 (format gate): books this version made +/// declared it as they were created, so the first change journals only +/// itself; a change to what is already set journals nothing. Books an +/// older version wrote in format 1 refuse the change, journaling nothing, +/// until their format upgrade (the owner's go-ahead, D-39); then it is +/// made, after the one declaration. #[test] -fn the_first_change_declares_format_2() { +fn a_change_needs_journal_format_2() { let (dir, mut s) = books("format"); - assert_eq!(s.journal_format().format, 1); + assert_eq!(s.journal_format().format, 2); let before = s.head().0; s.change_settings(&[set("company.name", json!("Acme LLC"))], &src("A")) .unwrap(); - assert_eq!(s.journal_format().format, 2); - assert_eq!(s.head().0, before + 2, "a declaration, then the change"); + assert_eq!(s.head().0, before + 1, "the change alone"); s.change_settings( &[set("company.timezone", json!("America/Denver"))], &src("A"), ) .unwrap(); - assert_eq!(s.head().0, before + 3, "declared once"); + assert_eq!(s.head().0, before + 2); let at = s.head().0; let made = s .change_settings(&[set("company.name", json!(" Acme LLC "))], &src("A")) @@ -134,7 +137,29 @@ fn the_first_change_declares_format_2() { assert!(made.is_empty()); assert_eq!(s.head().0, at, "nothing to change, nothing journaled"); let r = s.ledger().settings().get("company.name").unwrap(); - assert_eq!((r.seq, r.source.decided_by.as_str()), (before + 2, "A")); + assert_eq!((r.seq, r.source.decided_by.as_str()), (before + 1, "A")); + // Books in format 1, as an older version wrote them. + let old = test_dir("format-old"); + let mut o = + Store::create_in_format(&old, Currency::USD, &ledger_store::JournalFormat::default()) + .unwrap(); + let at = o.head().0; + for refused in [ + o.change_settings(&[set("company.name", json!("Acme LLC"))], &src("A")), + o.adopt_settings(&[], &src("A")), + ] { + assert!( + matches!(refused, Err(StoreError::FormatUpgradePending { .. })), + "{refused:?}" + ); + } + assert_eq!(o.head().0, at, "refused: nothing journaled"); + assert!(!o.ledger().settings().is_journaled()); + o.upgrade_format().unwrap().expect("upgraded"); + o.change_settings(&[set("company.name", json!("Acme LLC"))], &src("A")) + .unwrap(); + assert_eq!(o.head().0, at + 2, "the declaration, then the change"); + assert_every_reader_agrees(&old, &o); drop(s); let s = Store::open(&dir).unwrap(); assert_eq!(s.journal_format().format, 2); @@ -1070,19 +1095,19 @@ fn an_empty_import_still_moves_the_books_in() { let at = s.head().0; assert!(s.change_settings(&[], &src("A")).unwrap().is_empty()); assert_eq!(s.head().0, at, "an ordinary empty change: nothing"); - assert_eq!(s.journal_format().format, 1); + assert_eq!(s.journal_format().format, 2, "declared at creation"); assert!(!s.ledger().settings().is_journaled()); assert!(s.adopt_settings(&[], &src("A")).unwrap().is_empty()); - assert_eq!(s.head().0, at + 2, "a declaration, then the move"); + assert_eq!(s.head().0, at + 1, "the move alone"); assert!(s.ledger().settings().is_journaled()); - assert_eq!(s.ledger().settings().last_seq(), Some(at + 2)); + assert_eq!(s.ledger().settings().last_seq(), Some(at + 1)); for again in [ s.adopt_settings(&[], &src("A")).unwrap(), s.change_settings(&[], &src("A")).unwrap(), ] { assert!(again.is_empty()); } - assert_eq!(s.head().0, at + 2, "moved in once"); + assert_eq!(s.head().0, at + 1, "moved in once"); drop(s); let s = Store::open(&dir).unwrap(); assert!(s.ledger().settings().is_journaled()); diff --git a/crates/ledger-store/tests/snapshot.rs b/crates/ledger-store/tests/snapshot.rs index 77aee40..d129645 100644 --- a/crates/ledger-store/tests/snapshot.rs +++ b/crates/ledger-store/tests/snapshot.rs @@ -186,7 +186,7 @@ fn a_snapshot_checks_the_writers_database_and_files() { assert_eq!(snap.check_disk(), DiskCheck::Matches); assert_eq!(snap.verify_blobs(), (1, vec![])); // Someone edits the database behind the journal's back. - let conn = rusqlite::Connection::open(dir.join(ledger_store::DB_FILE)).unwrap(); + let conn = rusqlite::Connection::open(dir.join(ledger_store::db_file())).unwrap(); conn.execute( "UPDATE accounts SET name = 'Petty cash' WHERE code = '1000'", [], diff --git a/crates/ledger-store/tests/spec_store.rs b/crates/ledger-store/tests/spec_store.rs index d590558..6cab714 100644 --- a/crates/ledger-store/tests/spec_store.rs +++ b/crates/ledger-store/tests/spec_store.rs @@ -10,7 +10,7 @@ use chrono::NaiveDate; use ledger_core::{AccountId, AccountType, Currency, EntryId, Money, NewAsset, Side}; use ledger_store::journal::repair_torn_tail; -use ledger_store::{BlobStore, DB_FILE, JOURNAL_FILE, Snapshot, Store, StoreError}; +use ledger_store::{BlobStore, JOURNAL_FILE, Snapshot, Store, StoreError, db_file}; use rust_decimal::Decimal; use serde_json::Value; use sha2::{Digest, Sha256}; @@ -156,7 +156,7 @@ fn file_names(dir: &Path) -> Vec { fn remove_db(dir: &Path) { for suffix in ["", "-wal", "-shm"] { - let _ = std::fs::remove_file(dir.join(format!("{DB_FILE}{suffix}"))); + let _ = std::fs::remove_file(dir.join(format!("{}{suffix}", db_file()))); } } @@ -764,7 +764,7 @@ fn open_rebuilds_a_missing_database() { drop(b); remove_db(&dir); let s = Store::open(&dir).unwrap(); - assert!(dir.join(DB_FILE).exists()); + assert!(dir.join(db_file()).exists()); assert_eq!(s.ledger(), &live); assert_eq!( s.events_since(0, 1_000_000).unwrap().len(), @@ -788,7 +788,7 @@ fn open_keeps_a_corrupt_database_aside_and_rebuilds() { .take(8192) .copied() .collect(); - std::fs::write(dir.join(DB_FILE), &garbage).unwrap(); + std::fs::write(dir.join(db_file()), &garbage).unwrap(); let journal_before = jbytes(&dir); let s = Store::open(&dir).unwrap(); assert_eq!(s.ledger(), &live); @@ -796,7 +796,7 @@ fn open_keeps_a_corrupt_database_aside_and_rebuilds() { // Kept aside: some other file in the directory holds the garbage. let kept = tree(&dir) .into_iter() - .any(|(p, bytes)| p != Path::new(DB_FILE) && bytes == garbage); + .any(|(p, bytes)| p != Path::new(&db_file()) && bytes == garbage); assert!( kept, "corrupt db not kept aside; files: {:?}", @@ -815,8 +815,8 @@ fn open_keeps_a_corrupt_database_aside_and_rebuilds() { fn open_catches_up_a_database_behind_the_journal() { let dir = test_dir("open-behind"); drop(books(&dir)); - let old_db = std::fs::read(dir.join(DB_FILE)).unwrap(); - let old_wal = std::fs::read(dir.join(format!("{DB_FILE}-wal"))).ok(); + let old_db = std::fs::read(dir.join(db_file())).unwrap(); + let old_wal = std::fs::read(dir.join(format!("{}-wal", db_file()))).ok(); let mut s = Store::open(&dir).unwrap(); s.add_account("6300", "Meals", AccountType::Expense, None, false) .unwrap(); @@ -826,9 +826,9 @@ fn open_catches_up_a_database_behind_the_journal() { let live = s.ledger().clone(); drop(s); remove_db(&dir); - std::fs::write(dir.join(DB_FILE), &old_db).unwrap(); + std::fs::write(dir.join(db_file()), &old_db).unwrap(); if let Some(w) = old_wal { - std::fs::write(dir.join(format!("{DB_FILE}-wal")), w).unwrap(); + std::fs::write(dir.join(format!("{}-wal", db_file())), w).unwrap(); } let s = Store::open(&dir).unwrap(); assert_eq!(s.ledger(), &live); @@ -1395,7 +1395,7 @@ fn verify_detects_projection_divergence() { .attach_receipt(Some(b.e1), "r.pdf", "application/pdf", b"d", None) .unwrap(); { - let c = rusqlite::Connection::open(dir.join(DB_FILE)).unwrap(); + let c = rusqlite::Connection::open(dir.join(db_file())).unwrap(); let n = c.execute("DELETE FROM attachments", []).unwrap(); assert_eq!(n, 1); } @@ -1448,7 +1448,7 @@ fn heal_rebuilds_a_diverged_projection() { .attach_receipt(Some(b.e1), "r.pdf", "application/pdf", b"h", None) .unwrap(); { - let c = rusqlite::Connection::open(dir.join(DB_FILE)).unwrap(); + let c = rusqlite::Connection::open(dir.join(db_file())).unwrap(); c.execute("DELETE FROM attachments", []).unwrap(); } let journal = jbytes(&dir); diff --git a/crates/ledger-store/tests/store.rs b/crates/ledger-store/tests/store.rs index ae3fba0..9d1dce2 100644 --- a/crates/ledger-store/tests/store.rs +++ b/crates/ledger-store/tests/store.rs @@ -304,7 +304,7 @@ fn missing_database_heals_on_open() { let dir = test_dir("healopen"); let store = build_books(&dir); drop(store); - std::fs::remove_file(dir.join(ledger_store::DB_FILE)).unwrap(); + std::fs::remove_file(dir.join(ledger_store::db_file())).unwrap(); // No database at all: open rebuilds the projection from the journal. let store = Store::open(&dir).unwrap(); let tb = ledger_core::reports::trial_balance(store.ledger()); @@ -426,7 +426,7 @@ fn corrupt_projection_is_quarantined_and_rebuilt() { let _store = build_books(&dir); } // Corrupt the SQLite file with garbage. - let db_path = dir.join(ledger_store::DB_FILE); + let db_path = dir.join(ledger_store::db_file()); std::fs::write(&db_path, b"this is not a sqlite database at all").unwrap(); // Reopen heals from the journal instead of failing. let store = Store::open(&dir).unwrap(); @@ -435,9 +435,11 @@ fn corrupt_projection_is_quarantined_and_rebuilt() { .unwrap() .filter_map(|e| e.ok()) .filter(|e| { + // Kept beside it under its own name: wanderbooks.v{N}.corrupt.* + let stem = ledger_store::db_file().trim_end_matches(".db").to_string(); e.file_name() .to_string_lossy() - .starts_with("wanderbooks.corrupt.") + .starts_with(&format!("{stem}.corrupt.")) }) .collect(); assert_eq!(quarantined.len(), 1); @@ -838,11 +840,11 @@ fn vendor_1099_tags_replay_and_project() { // Reopen (replay), then rebuild the projection from scratch. for rebuild in [false, true] { if rebuild { - std::fs::remove_file(dir.join(ledger_store::DB_FILE)).unwrap(); + std::fs::remove_file(dir.join(ledger_store::db_file())).unwrap(); } let store = Store::open(&dir).unwrap(); assert_eq!(report(store.ledger()), before); - let conn = rusqlite::Connection::open(dir.join(ledger_store::DB_FILE)).unwrap(); + let conn = rusqlite::Connection::open(dir.join(ledger_store::db_file())).unwrap(); let tagged: i64 = conn .query_row( "SELECT COUNT(*) FROM entry_vendors WHERE vendor_id = ?", @@ -885,7 +887,7 @@ fn the_whole_projection_is_checked_and_healed() { store.verify().unwrap(); drop(store); // Drift no accounting report would notice: a receipt quietly unlinked. - let db = dir.join(ledger_store::DB_FILE); + let db = dir.join(ledger_store::db_file()); let conn = rusqlite::Connection::open(&db).unwrap(); conn.execute("UPDATE attachments SET entry_id = NULL", []) .unwrap(); @@ -1781,7 +1783,7 @@ fn books_written_under_older_rules_still_open() { .unwrap(); drop(w); // Opening rebuilds from the journal; it must accept all of it. - let _ = std::fs::remove_file(dir.join(ledger_store::DB_FILE)); + let _ = std::fs::remove_file(dir.join(ledger_store::db_file())); let store = Store::open(&dir).expect("older books still open"); store.verify().unwrap(); assert_eq!(store.journal_format().format, 1); @@ -2570,12 +2572,12 @@ fn the_viewer_tells_behind_from_ahead() { // Behind: a database restored from before that write. drop(store); let mut store = Store::open(&dir).unwrap(); - let before_write = std::fs::read(dir.join(ledger_store::DB_FILE)).unwrap(); + let before_write = std::fs::read(dir.join(ledger_store::db_file())).unwrap(); store.add_vendor("Supplier").unwrap(); drop(store); - std::fs::write(dir.join(ledger_store::DB_FILE), &before_write).unwrap(); - let _ = std::fs::remove_file(dir.join("wanderbooks.db-wal")); - let _ = std::fs::remove_file(dir.join("wanderbooks.db-shm")); + std::fs::write(dir.join(ledger_store::db_file()), &before_write).unwrap(); + let _ = std::fs::remove_file(dir.join(format!("{}-wal", ledger_store::db_file()))); + let _ = std::fs::remove_file(dir.join(format!("{}-shm", ledger_store::db_file()))); let snap = Snapshot::load(&dir).unwrap(); match snap.check_disk() { DiskCheck::Behind { applied, journal } => assert!(applied < journal), @@ -2590,7 +2592,7 @@ fn the_table_check_counts_missing_and_unexpected_rows() { let dir = test_dir("table-counts"); let store = build_books(&dir); drop(store); - let conn = rusqlite::Connection::open(dir.join(ledger_store::DB_FILE)).unwrap(); + let conn = rusqlite::Connection::open(dir.join(ledger_store::db_file())).unwrap(); // The events table mirrors the journal and isn't read back into the // ledger, so only the table check sees this. conn.execute( @@ -2636,7 +2638,7 @@ fn a_database_with_the_wrong_currency_is_rebuilt() { let store = build_books(&dir); let before = ledger_core::reports::snapshot(store.ledger()).unwrap(); drop(store); - let conn = rusqlite::Connection::open(dir.join(ledger_store::DB_FILE)).unwrap(); + let conn = rusqlite::Connection::open(dir.join(ledger_store::db_file())).unwrap(); conn.execute( "UPDATE meta SET value = 'EUR' WHERE key = 'base_currency'", [], @@ -2780,7 +2782,7 @@ fn the_database_tracks_what_an_invoice_has_been_paid() { ) .unwrap(); let row = |dir: &std::path::Path| -> (String, String) { - let conn = rusqlite::Connection::open(dir.join(ledger_store::DB_FILE)).unwrap(); + let conn = rusqlite::Connection::open(dir.join(ledger_store::db_file())).unwrap(); conn.query_row( "SELECT amount_paid, status FROM invoices WHERE id = ?", [inv as i64], @@ -2807,7 +2809,7 @@ fn rechain(dir: &std::path::Path, change: impl Fn(&mut Vec Vec<(String, Vec)> { - let conn = rusqlite::Connection::open(dir.join(ledger_store::DB_FILE)).unwrap(); + let conn = rusqlite::Connection::open(dir.join(ledger_store::db_file())).unwrap(); let mut tables: Vec = conn .prepare("SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name") .unwrap() @@ -3065,28 +3067,29 @@ fn rename_account_changes_only_the_name_defect_15() { .rename_account(rent, " Office Rent ", Some("owner asked 2026-10-21")) .unwrap(); let after = std::fs::read_to_string(&journal).unwrap(); - // Every earlier line kept, byte for byte; two lines appended: the - // feature's declaration, then the rename. + // Every earlier line kept, byte for byte; one line appended, the + // rename (its feature, account_renamed, the books declared as they + // were created, with the rest of this version's format). assert!(after.starts_with(&text)); + assert!( + store + .journal_format() + .has_feature(ledger_store::format::FEATURE_ACCOUNT_RENAMED) + ); let added: Vec = after[text.len()..] .lines() .map(|l| serde_json::from_str(l).unwrap()) .collect(); - assert_eq!(added.len(), 2); - assert_eq!(added[0]["type"], "journal.format_set"); + assert_eq!(added.len(), 1); + assert_eq!(added[0]["type"], "account.renamed"); assert_eq!( added[0]["payload"], - serde_json::json!({"format": 2, "features": ["account_renamed"]}) - ); - assert_eq!(added[1]["type"], "account.renamed"); - assert_eq!( - added[1]["payload"], serde_json::json!({ "account_id": rent, "code": "6000", "name": "Office Rent", "previous": "Rent Expense", "reason": "owner asked 2026-10-21" }) ); - assert_eq!(store.head().0, seq + 2); + assert_eq!(store.head().0, seq + 1); assert_eq!( store.ledger().chart().get(rent).unwrap().name, "Office Rent" @@ -3096,7 +3099,7 @@ fn rename_account_changes_only_the_name_defect_15() { // The one column a rename changes, read from the projection itself // (projection_but_names blanks it). let db_name = |dir: &std::path::Path| -> String { - rusqlite::Connection::open(dir.join(ledger_store::DB_FILE)) + rusqlite::Connection::open(dir.join(ledger_store::db_file())) .unwrap() .query_row( "SELECT name FROM accounts WHERE id = ?", @@ -3120,7 +3123,7 @@ fn rename_account_changes_only_the_name_defect_15() { assert_eq!(reopened.ledger().chart().get(rent).unwrap().name, "Rent"); assert_eq!(figures_but_names(reopened.ledger()), figures); drop(reopened); - let _ = std::fs::remove_file(dir.join(ledger_store::DB_FILE)); + let _ = std::fs::remove_file(dir.join(ledger_store::db_file())); let rebuilt = Store::open(&dir).unwrap(); assert_eq!(rebuilt.ledger().chart().get(rent).unwrap().name, "Rent"); assert_eq!(projection_but_names(&dir), rows); @@ -3159,7 +3162,7 @@ fn a_journaled_rename_replays_past_the_command_rules() { ) .unwrap(); drop(w); - let _ = std::fs::remove_file(dir.join(ledger_store::DB_FILE)); + let _ = std::fs::remove_file(dir.join(ledger_store::db_file())); let store = Store::open(&dir); assert_eq!(store.is_ok(), opens, "{name:?}"); let store = store.unwrap(); @@ -3181,7 +3184,7 @@ fn a_journaled_rename_replays_past_the_command_rules() { ) .unwrap(); drop(w); - let _ = std::fs::remove_file(dir.join(ledger_store::DB_FILE)); + let _ = std::fs::remove_file(dir.join(ledger_store::db_file())); assert!(Store::open(&dir).is_err()); } @@ -3207,8 +3210,9 @@ fn tax_policy(store: &mut Store, value: Option<&str>) { /// expense account that isn't a header is refused without one, journaling /// nothing, and created with one as account.added then account.tax_set; /// a needed category can't be cleared; the accounts the engine creates -/// for a control role are never refused. The first setting of the key -/// declares the feature require_tax_category, once. +/// for a control role are never refused. Setting the key needs the feature +/// require_tax_category, which new books declared as they were created: +/// the setting journals only itself. #[test] fn the_tax_category_policy_refuses_only_what_it_says() { let dir = test_dir("tax-policy"); @@ -3224,12 +3228,8 @@ fn the_tax_category_policy_refuses_only_what_it_says() { .lines() .map(|l| serde_json::from_str(l).unwrap()) .collect(); - assert_eq!(new[0]["type"], "journal.format_set"); - assert_eq!( - new[0]["payload"], - serde_json::json!({"format": 2, "features": ["require_tax_category"]}) - ); - assert_eq!(new.last().unwrap()["type"], "settings.set"); + assert_eq!(new.len(), 1); + assert_eq!(new[0]["type"], "settings.set"); for (code, t, header, tax, refused) in [ ("6160", AccountType::Expense, false, None, true), ("4100", AccountType::Revenue, false, None, true), @@ -3375,7 +3375,7 @@ fn a_journaled_clear_replays_under_refuse() { ) .unwrap(); drop(w); - let _ = std::fs::remove_file(dir.join(ledger_store::DB_FILE)); + let _ = std::fs::remove_file(dir.join(ledger_store::db_file())); let store = Store::open(&dir).unwrap(); store.verify().unwrap(); assert!( @@ -3646,7 +3646,7 @@ fn books_written_under_p3_keep_their_names() { ] ); drop(store); - let _ = std::fs::remove_file(dir.join(ledger_store::DB_FILE)); + let _ = std::fs::remove_file(dir.join(ledger_store::db_file())); let rebuilt = Store::open(&dir).unwrap(); assert_eq!(chart_of(rebuilt.ledger()), want); drop(rebuilt); @@ -3683,10 +3683,10 @@ fn a_new_accounts_name_is_journaled_trimmed() { assert_eq!(store.ledger().chart().get(id).unwrap().name, "Hosting"); let live = store.ledger().clone(); drop(store); - let _ = std::fs::remove_file(dir.join(ledger_store::DB_FILE)); + let _ = std::fs::remove_file(dir.join(ledger_store::db_file())); let reopened = Store::open(&dir).unwrap(); assert!(live.differs_from(reopened.ledger()).is_empty()); - let conn = rusqlite::Connection::open(dir.join(ledger_store::DB_FILE)).unwrap(); + let conn = rusqlite::Connection::open(dir.join(ledger_store::db_file())).unwrap(); let name: String = conn .query_row("SELECT name FROM accounts WHERE id = ?", [id as i64], |r| { r.get(0) diff --git a/docs/mcp-interface.md b/docs/mcp-interface.md index d4caa5e..5ebc474 100644 --- a/docs/mcp-interface.md +++ b/docs/mcp-interface.md @@ -129,8 +129,13 @@ settings-and-conventions section says how to use them. state ("needs_onboarding" or "active"), settings_source, rules: { evidence, parties, approval_rules }, settings_summary: [{ key, value, set_by, seq }], conventions: [{ id, kind, title, summary }], open_items: - [{ id, title, due }], truncated, handbook_sections, next_steps: [{ tool, - args?, why }], isolation }`. The connect text names it as the first call. + [{ id, title, due }], truncated, handbook_sections, format_upgrade?: { + guard, format, adds, how }, next_steps: [{ tool, args?, why }], + isolation }`. The connect text names it as the first call. + `format_upgrade` is there only for books waiting for a format upgrade + (below); its next step comes first, before anything else, at every + start until it is done: `books.handbook {section: "format-upgrade"}`, + "Tell the human first: …". `isolation` is `same_user (advisory)` (D-32): the books are hidden from agents in the owner's own account, not locked against them. Lists cut to fit say so in `truncated` ("conventions: 4 more (books.conventions)"). @@ -138,6 +143,25 @@ settings-and-conventions section says how to use them. company's close steps, `audit` `books.verify`. Not built yet (P5): `book.id`, `book.url`, `head.last_handoff`, the `agent` input, and loading it up front under tool search. Also `books://start`. +**The format upgrade** (plan 3.9, D-39, rules.md B-27). Books an older +wanderbooks wrote are in an older journal format: they are served on it, +and every tool that needs nothing new works as before. A tool whose +change needs the newer format (on main: settings, conventions, decisions, +open items and approvals need journal format 2; later formats add +features) is refused before anything is journaled, its error beginning +`format_upgrade_pending:` and saying that only the owner can give the +go-ahead, with the owner passphrase, in the app or with `wanderbooks +books upgrade ID` at a terminal, and that no tool can. No tool consents: +none is offered, and the server takes the go-ahead only on its admin +socket, from the host's lease intent `consent_format_upgrade` (the app's +dialog, `books upgrade`), with the passphrase it checks. Before the +upgrade the server commits the books to their own backup repository and +proves the copy holds them exactly (journal, stored files, vault, keys); +then one `journal.format_set` declares the whole format, and the same +session's next call can make the change. Books this version creates are +in its format from their second line, and an adoption's copy moves to it +as it is made. + - `books.conventions { kind?, include_retired? }` → every convention with its full text, structure, revision and source (`books://conventions`). `books.set_convention { id, kind, title, body_md, structured?, @@ -746,11 +770,14 @@ through the policy. ## Tools — integrity - `books.verify { blobs? }` → `{ status: "ok" | "healed", events, head, - healed?, blobs_verified?, checked }`. Verifies the journal's hash chain, + healed?, blobs_verified?, database, other_databases?, checked }` + (`database`: the projection file verified, one for each schema, + `wanderbooks.v{N}.db`; `other_databases`: other schemas' files beside it, + left for the versions that use them). Verifies the journal's hash chain, replays it against the live ledger, and compares every table of the SQLite database row for row against a fresh projection of the journal. A drifted database is rebuilt from the journal (the old file kept as - `wanderbooks.mismatch.