diff --git a/crates/ledger-gui/src/data.rs b/crates/ledger-gui/src/data.rs index b88ebc3..d76d6de 100644 --- a/crates/ledger-gui/src/data.rs +++ b/crates/ledger-gui/src/data.rs @@ -8,7 +8,7 @@ use crate::books::{Books, Integrity}; use chrono::NaiveDate; use ledger_config::BooksSettings; -use ledger_store::Snapshot; +use ledger_store::{Snapshot, StoreError}; use notify::{RecursiveMode, Watcher}; use std::collections::HashSet; use std::path::PathBuf; @@ -62,11 +62,32 @@ impl Loader { /// Load once, synchronously (tests and one-shot tools). pub fn load_once(source: &Source) -> Result { - let snap = Snapshot::load(&source.data_dir).map_err(|e| e.to_string())?; + let snap = Snapshot::load(&source.data_dir).map_err(|e| { + problem(&e, || { + format!("Can't read the books in {}", source.data_dir.display()) + }) + })?; let integrity = Integrity::check(&snap, &mut HashSet::new()); Books::build(&snap, settings(source)?, today(source), None, integrity) } +/// What the viewer says when the books can't be read. Books from a newer +/// wanderbooks aren't broken, so they get the plain "update" message on its +/// own, never a failure framing; anything else is `context: error`. +pub fn problem(e: &StoreError, context: impl FnOnce() -> String) -> String { + match e { + StoreError::NewerJournal { .. } => { + let text = e.to_string(); + let mut chars = text.chars(); + chars + .next() + .map(|c| c.to_uppercase().chain(chars).collect()) + .unwrap_or_default() + } + _ => format!("{}: {e}", context()), + } +} + fn today(source: &Source) -> NaiveDate { source .today @@ -95,10 +116,9 @@ fn run(source: Source, ctx: egui::Context, tx: Sender) { match Snapshot::load(&source.data_dir) { Ok(s) => break s, Err(e) => { - send(Update::Error(format!( - "Can't read the books in {}: {e}", - source.data_dir.display() - ))); + send(Update::Error(problem(&e, || { + format!("Can't read the books in {}", source.data_dir.display()) + }))); std::thread::sleep(Duration::from_secs(2)); } } @@ -178,7 +198,9 @@ fn run(source: Source, ctx: egui::Context, tx: Sender) { let grew = match snap.refresh() { Ok(n) => n > 0, Err(e) => { - send(Update::Error(format!("Reading the journal failed: {e}"))); + send(Update::Error(problem(&e, || { + "Reading the journal failed".into() + }))); continue; } }; diff --git a/crates/ledger-gui/src/pages/journal.rs b/crates/ledger-gui/src/pages/journal.rs index fb272a4..e641df1 100644 --- a/crates/ledger-gui/src/pages/journal.rs +++ b/crates/ledger-gui/src/pages/journal.rs @@ -785,6 +785,11 @@ pub fn describe(b: &Books, line: &JournalLine) -> Option { format!("Cut-off journal line set aside ({} bytes)", p["bytes"]), None, ), + "journal.format_set" => ev( + icon::FILE_TEXT, + format!("Journal format {} declared", p["format"]), + None, + ), "recurring.generated" => ev( icon::REPEAT, "Recurring draft created".into(), diff --git a/crates/ledger-gui/tests/newer_journal.rs b/crates/ledger-gui/tests/newer_journal.rs new file mode 100644 index 0000000..2cfe289 --- /dev/null +++ b/crates/ledger-gui/tests/newer_journal.rs @@ -0,0 +1,164 @@ +//! Books written by a newer wanderbooks: the viewer says to update, never +//! that the books are corrupt, on its first load and while following them. +//! +//! ```text +//! GOAL: Stephen, Kat or the CPA, on an older viewer, learn to update it, +//! not that the books are damaged. +//! SOURCE: rules.md B-27; plan/served.md 5.1 (the viewer reads the journal +//! directly, so it meets the same gate). +//! ``` + +use chrono::NaiveDate; +use ledger_core::{AccountType, Currency}; +use ledger_gui::data::{self, Loader, Source, Update}; +use ledger_store::{JOURNAL_FILE, JournalWriter, Store}; +use serde_json::json; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +fn books(tag: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("ledger-gui-gate-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let data = dir.join("books-data"); + let mut store = Store::create(&data, Currency::USD).unwrap(); + store + .add_account("1000", "Checking", AccountType::Asset, None, false) + .unwrap(); + data +} + +fn source(data: &Path) -> Source { + Source { + data_dir: data.to_path_buf(), + books_toml: None, + today: NaiveDate::from_ymd_opt(2026, 10, 16), + } +} + +fn rate() -> serde_json::Value { + json!({"currency": "EUR", "date": "2026-10-01", "rate": "1.10"}) +} + +/// The last line can't be checked: the viewer says so, and never that the +/// books are not damaged (rules.md B-27). +fn says_newer_or_damaged(text: &str) { + assert!(text.starts_with("Journal line"), "{text}"); + assert!(text.contains("or the line is damaged"), "{text}"); + assert!(text.contains("Update wanderbooks"), "{text}"); + assert!(!text.contains("not damaged"), "{text}"); +} + +fn says_update(text: &str) { + assert!( + text.starts_with("These books were written by a newer version of wanderbooks"), + "{text}" + ); + assert!(text.contains("Update wanderbooks"), "{text}"); + assert!(!text.contains("corrupt"), "{text}"); +} + +/// Opening: a newer event type, or a newer declared format, reads as +/// "update"; a damaged journal still reads as corrupt. +#[test] +fn the_viewer_says_update_for_newer_books_and_corrupt_for_damaged_ones() { + for (tag, line_type, payload) in [ + ("type", "settings.set", json!({"key": "company.name"})), + ( + "format", + "journal.format_set", + json!({"format": 2, "features": []}), + ), + ] { + let data = books(tag); + let mut w = JournalWriter::open(&data.join(JOURNAL_FILE)).unwrap(); + w.append(line_type, payload.clone()).unwrap(); + // The next line chains onto it: its bytes are checked. + w.append("fx.rate_set", rate()).unwrap(); + let err = data::load_once(&source(&data)) + .err() + .expect("newer books loaded"); + says_update(&err); + + // As the last line nothing checks it: newer, or damaged. + let data = books(&format!("{tag}-last")); + JournalWriter::open(&data.join(JOURNAL_FILE)) + .unwrap() + .append(line_type, payload) + .unwrap(); + let err = data::load_once(&source(&data)) + .err() + .expect("newer books loaded"); + says_newer_or_damaged(&err); + } + + let data = books("damaged"); + let path = data.join(JOURNAL_FILE); + let text = std::fs::read_to_string(&path).unwrap(); + std::fs::write(&path, text.replacen("\"prev\":\"", "\"prev\":\"1", 2)).unwrap(); + let err = data::load_once(&source(&data)) + .err() + .expect("damaged books loaded"); + assert!(err.contains("journal corrupt"), "{err}"); + assert!(!err.contains("newer"), "{err}"); +} + +/// Following: the viewer has the books open when a newer wanderbooks +/// writes a line this one can't read. It says to update, and keeps the +/// books it had. +#[test] +fn a_live_viewer_says_update_when_the_books_move_ahead_of_it() { + let data = books("live"); + let loader = Loader::spawn(source(&data), egui::Context::default()); + match loader.wait(Duration::from_secs(10)) { + Some(Update::Books(_)) => {} + _ => panic!("no initial load"), + } + let mut w = JournalWriter::open(&data.join(JOURNAL_FILE)).unwrap(); + w.append("entry.evidence_inherited", json!({"entry_id": 1})) + .unwrap(); + w.append("fx.rate_set", rate()).unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + let mut said = None; + while Instant::now() < deadline { + match loader.wait(Duration::from_millis(200)) { + Some(Update::Error(e)) => { + said = Some(e); + break; + } + Some(Update::Books(_)) => panic!("the viewer read a line it doesn't know"), + _ => {} + } + } + says_update(&said.expect("the viewer never said to update")); +} + +/// A declared format this version reads opens like any books, and the +/// journal page tells it as its own event (it is no step of an entry). +#[test] +fn a_format_this_version_reads_opens_and_the_journal_page_tells_it() { + let data = books("current-format"); + let mut w = JournalWriter::open(&data.join(JOURNAL_FILE)).unwrap(); + w.append( + ledger_store::format::FORMAT_SET, + json!({"format": ledger_store::format::SUPPORTED_JOURNAL_FORMAT, "features": []}), + ) + .unwrap(); + w.append("fx.rate_set", rate()).unwrap(); + let b = data::load_once(&source(&data)).expect("books in a format this version reads"); + let told: Vec<_> = b + .journal + .iter() + .filter(|l| l.line_type == ledger_store::format::FORMAT_SET) + .map(|l| ledger_gui::pages::journal::describe(&b, l).map(|e| (e.text, e.route))) + .collect(); + assert_eq!( + told, + [Some(( + format!( + "Journal format {} declared", + ledger_store::format::SUPPORTED_JOURNAL_FORMAT + ), + None + ))] + ); +} diff --git a/crates/ledger-mcp/src/main.rs b/crates/ledger-mcp/src/main.rs index f6b151e..fc545b0 100644 --- a/crates/ledger-mcp/src/main.rs +++ b/crates/ledger-mcp/src/main.rs @@ -112,7 +112,16 @@ fn main() -> Result<(), Box> { let live = std::sync::Arc::new(std::sync::Mutex::new(live)); let mut store = if data_dir.join(ledger_store::JOURNAL_FILE).exists() { - open_waiting_for_lock(&data_dir)? + match open_waiting_for_lock(&data_dir) { + Ok(store) => store, + // Books from a newer wanderbooks are fine; this server is old. + // Say so in plain words (the error's text), and leave them be. + Err(e @ ledger_store::StoreError::NewerJournal { .. }) => { + eprintln!("ledger-mcp: {e}"); + std::process::exit(1); + } + Err(e) => return Err(e.into()), + } } else { let code = config.base_currency.as_deref().unwrap_or("USD"); let currency = Currency::new(code).map_err(|_| format!("bad base currency {code:?}"))?; diff --git a/crates/ledger-mcp/tests/startup.rs b/crates/ledger-mcp/tests/startup.rs index a157a1b..2f13abd 100644 --- a/crates/ledger-mcp/tests/startup.rs +++ b/crates/ledger-mcp/tests/startup.rs @@ -172,3 +172,42 @@ fn a_held_lock_is_refused_not_waited_on_forever() { "gave up before waiting" ); } + +/// Books written by a newer wanderbooks: the server won't start on them, +/// and says to update it, not that the books are corrupt (rules.md B-27). +#[test] +fn newer_books_are_refused_as_newer_not_corrupt() { + let dir = temp("newer"); + drop(ledger_store::Store::create(&dir, ledger_core::Currency::USD).unwrap()); + let mut w = ledger_store::JournalWriter::open(&dir.join(ledger_store::JOURNAL_FILE)).unwrap(); + w.append( + "journal.format_set", + serde_json::json!({"format": 2, "features": []}), + ) + .unwrap(); + // The next line chains onto it, so its bytes are checked (B-27). + w.append( + "fx.rate_set", + serde_json::json!({"currency": "EUR", "date": "2026-10-01", "rate": "1.10"}), + ) + .unwrap(); + let journal = std::fs::read(dir.join(ledger_store::JOURNAL_FILE)).unwrap(); + let out = Command::new(BIN) + .arg("--data-dir") + .arg(&dir) + .stdin(Stdio::null()) + .output() + .unwrap(); + assert!(!out.status.success()); + let err = String::from_utf8_lossy(&out.stderr); + assert!( + err.contains("written by a newer version of wanderbooks"), + "{err}" + ); + assert!(err.contains("Update wanderbooks"), "{err}"); + assert!(!err.contains("corrupt"), "{err}"); + assert_eq!( + std::fs::read(dir.join(ledger_store::JOURNAL_FILE)).unwrap(), + journal + ); +} diff --git a/crates/ledger-store/src/error.rs b/crates/ledger-store/src/error.rs index 3c3ee77..f687245 100644 --- a/crates/ledger-store/src/error.rs +++ b/crates/ledger-store/src/error.rs @@ -55,6 +55,33 @@ pub enum StoreError { #[error("journal corrupt: {0}")] JournalCorrupt(String), + /// A whole, correctly chained journal line this version can't read: + /// a newer wanderbooks wrote the books. When the line after it chains + /// onto it (`verified`), its own bytes are checked: it is intact, + /// though the lines after it are the newer version's to check. When + /// nothing checks it (it is the last line, or the next has no `prev` + /// this version can read), it is either newer or damaged, and the + /// message says so (rules.md B-27). + #[error("{}", newer_journal_text(*seq, event_type, need, *verified))] + NewerJournal { + /// The first line this version can't read. + seq: u64, + /// Its type. + event_type: String, + /// What reading it needs: a journal format, features, or an event + /// type this version doesn't know. + need: String, + /// Whether the line's own hash was checked (the next line's `prev` + /// matches it). False when nothing checks it: a damaged line would + /// read the same. + verified: bool, + }, + + /// A journal format this version can't declare: newer than it reads, + /// lower than the books already use, or with a feature it doesn't know. + #[error("{0}")] + FormatRefused(String), + /// Replay produced different ids than the journal recorded. #[error("replay mismatch: {0}")] ReplayMismatch(String), @@ -109,3 +136,26 @@ pub enum StoreError { #[error("base currency mismatch: store is {0}, requested {1}")] CurrencyMismatch(String, String), } + +/// What [`StoreError::NewerJournal`] says. Only a line whose own hash was +/// checked lets it say that line is intact; it never vouches for the +/// lines after it, which this version can't read. +fn newer_journal_text(seq: u64, event_type: &str, need: &str, verified: bool) -> String { + let format = crate::format::SUPPORTED_JOURNAL_FORMAT; + if verified { + format!( + "these books were written by a newer version of wanderbooks: journal line {seq} \ + ({event_type}) needs {need}, and this version reads journal format {format}. \ + Update wanderbooks to open them. Line {seq} is intact (the line after it checks \ + it); this version can't check the lines after it" + ) + } else { + format!( + "journal line {seq} ({event_type}) needs {need}, and this version reads journal \ + format {format}. Nothing this version can read checks that line (it is the last \ + line, or the line after it isn't one this version reads): either a newer version \ + of wanderbooks wrote it, or the line is damaged. Update wanderbooks, then run \ + verify to tell which" + ) + } +} diff --git a/crates/ledger-store/src/format.rs b/crates/ledger-store/src/format.rs new file mode 100644 index 0000000..fa544a4 --- /dev/null +++ b/crates/ledger-store/src/format.rs @@ -0,0 +1,469 @@ +//! The format gate: what this version of wanderbooks can read. +//! +//! Every journal line has a type, and this version knows a fixed set of +//! them ([`EVENT_TYPES`]). Books written by a newer wanderbooks can hold a +//! type this version has never seen, or declare a newer journal format +//! (`journal.format_set`) before the first event that needs it. Either way +//! the line is whole and chained: the books are fine, and this reader is +//! old. It says so ([`StoreError::NewerJournal`]) instead of calling the +//! books corrupt, and stops there: it never skips a line it can't read. +//! +//! The gate runs as each line is read, after its parse, seq and hash +//! checks ([`crate::journal::read_journal_and_format`], and +//! [`crate::Snapshot`] line by line). +//! +//! What every format shares is frozen (rules.md B-27): a line is the +//! envelope `{seq, ts, type, payload, prev}` and nothing beside it, and +//! its hash is the SHA-256 of its canonical JSON (keys sorted, no +//! whitespace). A newer format adds its fields inside `payload`. So this +//! version can always follow the chain one line past the first line it +//! can't read, and say whether that line is intact. + +use crate::error::StoreError; +use crate::journal::{JournalLine, ReadLine}; +use serde_json::{Value, json}; + +/// The newest journal format this version reads and writes. +pub const SUPPORTED_JOURNAL_FORMAT: u32 = 1; + +/// Optional journal features this version reads (none yet). +pub const KNOWN_FEATURES: &[&str] = &[]; + +/// The line type that declares the journal format. +pub const FORMAT_SET: &str = "journal.format_set"; + +/// Every journal line type this version reads. Each has a replay arm and a +/// projection arm in `store.rs` (`tests/format.rs` checks both ways). +pub const EVENT_TYPES: &[&str] = &[ + "account.added", + "account.renumbered", + "account.tax_set", + "asset.acquisition_linked", + "asset.corrected", + "asset.depreciation_posted", + "asset.described", + "asset.disposed", + "asset.registered", + "asset.serial_set", + "asset.tagged", + "asset.tax_basis_set", + "attachment.linked", + "attachment.unlinked", + "attachment.withdrawn", + "bill.created", + "bill.paid", + "class.added", + "customer.added", + "document.filed", + "entry.created", + "entry.line_added", + "entry.line_class_set", + "entry.no_party", + "entry.parties_set", + "entry.party_set", + "entry.posted", + "entry.reversed", + "entry.vendor_set", + "fx.rate_set", + "invoice.created", + "invoice.payment_received", + FORMAT_SET, + "journal.tail_repaired", + "line.cleared", + "party.added", + "party.role_set", + "party.tin_set", + "party.updated", + "party.w4_set", + "party.w9_set", + "payroll.run", + "payroll.taxes_remitted", + "period.closed", + "period.created", + "receipt.attached", + "receipt.linked", + "receipt.unlinked", + "reconciliation.completed", + "reconciliation.voided", + "recurring.generated", + "recurring.registered", + "role.assigned", + "schedule.posted", + "schedule.registered", + "statement.recorded", + "statement.voided", + "store.created", + "vendor.1099_set", + "vendor.added", +]; + +/// The journal format in force: the last `journal.format_set` read, or +/// format 1 with no features for books that never declared one. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct JournalFormat { + /// Format number: 1 for every journal written before the gate. + pub format: u32, + /// Optional features in use, as declared. + pub features: Vec, +} + +impl Default for JournalFormat { + fn default() -> Self { + Self { + format: 1, + features: Vec::new(), + } + } +} + +impl JournalFormat { + /// Pass one verified line through the gate, before it is replayed. + /// + /// ```text + /// GOAL: an older wanderbooks never misreads newer books, and never + /// calls them corrupt. + /// SOURCE: rules.md B-27; plan/served.md 5.1 (format gate). + /// INVARIANTS: + /// 1. Only lines that parsed, chained and kept their seq reach the + /// gate: damage stays JournalCorrupt. + /// 2. NEVER admits a line this version can't replay. + /// SCENARIOS: + /// 1. IF the type is one this version doesn't know: SHOULD refuse + /// with NewerJournal + /// BECAUSE skipping it would show books that differ from the + /// record without saying so. + /// 2. IF it is a journal.format_set newer than + /// SUPPORTED_JOURNAL_FORMAT, or with a feature not in + /// KNOWN_FEATURES: SHOULD refuse with NewerJournal. + /// 3. IF it is a journal.format_set this version reads: SHOULD + /// record it as the format in force. + /// 4. IF a journal.format_set payload is malformed: SHOULD refuse + /// with JournalCorrupt. + /// ``` + pub fn admit(&mut self, line: &JournalLine) -> Result<(), StoreError> { + if line.line_type == FORMAT_SET { + *self = declared(line)?; + Ok(()) + } else if EVENT_TYPES.contains(&line.line_type.as_str()) { + Ok(()) + } else { + Err(unknown_type(line)) + } + } + + /// Check a declaration this version is about to write: a format it + /// reads, not lower than the one in force, with features it knows. + /// Returns the journal payload. + pub(crate) fn declaration(&self, format: u32, features: &[&str]) -> Result { + if format == 0 || format > SUPPORTED_JOURNAL_FORMAT { + return Err(StoreError::FormatRefused(format!( + "journal format {format}: this version writes 1 through {SUPPORTED_JOURNAL_FORMAT}" + ))); + } + if format < self.format { + return Err(StoreError::FormatRefused(format!( + "journal format {format}: the books already use format {}, and a format is never lowered", + self.format + ))); + } + if let Some(f) = features.iter().find(|f| !KNOWN_FEATURES.contains(f)) { + return Err(StoreError::FormatRefused(format!( + "journal feature {f:?}: this version doesn't know it" + ))); + } + let mut features: Vec<&str> = features.to_vec(); + features.sort_unstable(); + features.dedup(); + Ok(json!({ "format": format, "features": features })) + } +} + +/// Read a `journal.format_set` line, refusing a format or feature this +/// version doesn't read. +pub(crate) fn declared(line: &JournalLine) -> Result { + let p = &line.payload; + let format = p + .get("format") + .and_then(Value::as_u64) + .filter(|&f| f > 0) + .ok_or_else(|| { + StoreError::JournalCorrupt(format!("seq {}: {FORMAT_SET} format", line.seq)) + })?; + // A newer format may shape the rest of its declaration differently: + // it is newer, whatever its features look like. + if format > u64::from(SUPPORTED_JOURNAL_FORMAT) { + return Err(newer(line, format!("journal format {format}"))); + } + let format = u32::try_from(format).unwrap_or(SUPPORTED_JOURNAL_FORMAT); + let features = p + .get("features") + .and_then(Value::as_array) + .and_then(|a| { + a.iter() + .map(|f| f.as_str().map(str::to_string)) + .collect::>>() + }) + .ok_or_else(|| { + StoreError::JournalCorrupt(format!("seq {}: {FORMAT_SET} features", line.seq)) + })?; + let unknown: Vec<&str> = features + .iter() + .map(String::as_str) + .filter(|f| !KNOWN_FEATURES.contains(f)) + .collect(); + if !unknown.is_empty() { + return Err(newer( + line, + format!("journal features {}", unknown.join(", ")), + )); + } + Ok(JournalFormat { format, features }) +} + +/// The refusal for a line whose type this version doesn't know. +pub(crate) fn unknown_type(line: &JournalLine) -> StoreError { + newer(line, "an event type this version doesn't know".into()) +} + +/// The gate for one line as read: a line with a key beside the envelope's +/// is newer (rules.md B-27: this version never drops a key it can't +/// read), and any other goes to `admit` (JournalFormat::admit). +pub(crate) fn gate( + read: &ReadLine, + admit: &mut impl FnMut(&JournalLine) -> Result<(), StoreError>, +) -> Result<(), StoreError> { + if read.unknown_keys.is_empty() { + admit(&read.line) + } else { + Err(newer( + &read.line, + format!( + "journal line fields this version doesn't know ({})", + read.unknown_keys.join(", ") + ), + )) + } +} + +/// The gate's refusal, unverified: only the reader knows whether a later +/// line checks this one's hash ([`settle`]). +fn newer(line: &JournalLine, need: String) -> StoreError { + StoreError::NewerJournal { + seq: line.seq, + event_type: line.line_type.clone(), + need, + verified: false, + } +} + +/// Settle the gate's refusal of a line whose hash is `hash`, given the raw +/// journal line after it (`next`, None when the refused line is the last). +/// +/// ```text +/// GOAL: damage is never reported as books from a newer wanderbooks, and +/// books from a newer wanderbooks never as damage. +/// SOURCE: rules.md B-27 (the envelope and its hash are frozen across +/// formats); docs/receipt-store.md "Line format" (a line's hash is +/// checked only by the next line's prev). +/// INVARIANTS: +/// 1. Only a NewerJournal refusal is settled; anything else passes +/// through unchanged. +/// 2. Of the next line only `prev` is read, leniently (any JSON object +/// with a string `prev`): it is written in a format this version +/// may not read, with floats, a string seq or keys of its own. +/// SCENARIOS: +/// 1. IF the next line has a prev that isn't `hash`: SHOULD be +/// JournalCorrupt +/// BECAUSE the refused line changed after it was written (a flipped +/// bit in its type reads like a type this version doesn't know). +/// 2. IF the next line's prev is `hash`: SHOULD be NewerJournal, +/// verified: the refused line is intact (the lines after it are the +/// newer version's to check). +/// 3. IF there is no next line, or it has no prev this version can +/// read: SHOULD be NewerJournal, unverified +/// BECAUSE nothing checks the refused line's own bytes, so the +/// message says "newer or damaged", never "intact". +/// ``` +pub(crate) fn settle(refusal: StoreError, hash: &str, next: Option<&[u8]>) -> StoreError { + let StoreError::NewerJournal { + seq, + event_type, + need, + .. + } = refusal + else { + return refusal; + }; + let prev = next + .and_then(|raw| serde_json::from_slice::(raw).ok()) + .and_then(|v| v.get("prev").and_then(Value::as_str).map(str::to_string)); + match prev { + Some(prev) if prev != hash => { + StoreError::JournalCorrupt(format!("hash chain broken at seq {}", seq + 1)) + } + found => StoreError::NewerJournal { + seq, + event_type, + need, + verified: found.is_some(), + }, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::store::{Store, apply_line_to_sqlite, replay_line_into_ledger}; + use ledger_core::{Currency, Ledger}; + + fn line(line_type: &str, payload: Value) -> JournalLine { + JournalLine { + seq: 7, + ts: "2026-10-16T00:00:00Z".into(), + line_type: line_type.into(), + payload, + prev: String::new(), + } + } + + fn is_newer(r: &Result<(), StoreError>) -> bool { + matches!(r, Err(StoreError::NewerJournal { .. })) + } + + /// The list and the code agree: every type the gate admits has a + /// replay arm and a projection arm (a known type with an empty payload + /// fails as corrupt, never as newer). + #[test] + fn every_known_event_type_has_replay_and_projection_arms() { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + Store::migrate(&conn).unwrap(); + for &t in EVENT_TYPES { + let l = line(t, json!({})); + let mut ledger = Ledger::new(Currency::USD); + let replayed = replay_line_into_ledger(&mut ledger, &l, Currency::USD); + assert!(!is_newer(&replayed), "{t}: no replay arm"); + assert!( + !is_newer(&apply_line_to_sqlite(&conn, &l)), + "{t}: no projection arm" + ); + } + let l = line("settings.set", json!({})); + let mut ledger = Ledger::new(Currency::USD); + assert!(is_newer(&replay_line_into_ledger( + &mut ledger, + &l, + Currency::USD + ))); + assert!(is_newer(&apply_line_to_sqlite(&conn, &l))); + } + + /// A declaration never lowers the format in force, and never names a + /// format or feature this version can't read back. + #[test] + fn a_declared_format_is_never_lowered_or_unreadable() { + let at1 = JournalFormat::default(); + assert_eq!( + at1.declaration(1, &[]).unwrap(), + json!({"format": 1, "features": []}) + ); + for (format, features) in [(0, &[][..]), (2, &[][..]), (1, &["x"][..])] { + assert!( + matches!( + at1.declaration(format, features), + Err(StoreError::FormatRefused(_)) + ), + "{format} {features:?}" + ); + } + let at2 = JournalFormat { + format: 2, + features: Vec::new(), + }; + assert!(matches!( + at2.declaration(1, &[]), + Err(StoreError::FormatRefused(m)) if m.contains("never lowered") + )); + } + + /// What the gate records, and what it refuses, line by line. + #[test] + fn the_gate_records_a_readable_format_and_refuses_a_newer_one() { + let mut f = JournalFormat::default(); + f.admit(&line(FORMAT_SET, json!({"format": 1, "features": []}))) + .unwrap(); + assert_eq!(f, JournalFormat::default()); + f.admit(&line("entry.posted", json!({}))).unwrap(); + match f.admit(&line(FORMAT_SET, json!({"format": 2, "features": []}))) { + Err(StoreError::NewerJournal { + seq, + event_type, + need, + verified, + }) => { + // The gate alone never vouches for the line's bytes. + assert!(!verified); + assert_eq!((seq, event_type.as_str()), (7, FORMAT_SET)); + assert_eq!(need, "journal format 2"); + } + other => panic!("expected NewerJournal, got {other:?}"), + } + match f.admit(&line( + FORMAT_SET, + json!({"format": 1, "features": ["a", "b"]}), + )) { + Err(StoreError::NewerJournal { need, .. }) => { + assert_eq!(need, "journal features a, b") + } + other => panic!("expected NewerJournal, got {other:?}"), + } + // A refused line leaves the format in force alone. + assert_eq!(f, JournalFormat::default()); + for bad in [ + json!({"features": []}), + json!({"format": 0, "features": []}), + json!({"format": "1", "features": []}), + json!({"format": 1}), + json!({"format": 1, "features": [1]}), + json!({"format": 1, "features": "a"}), + ] { + assert!( + matches!( + f.admit(&line(FORMAT_SET, bad.clone())), + Err(StoreError::JournalCorrupt(_)) + ), + "{bad}" + ); + } + } + + /// A format newer than this version reads is newer whatever shape the + /// rest of its declaration takes, and however large its number: a + /// newer format may define its features differently, so they are + /// judged only for a format this version reads (rules.md B-27). + #[test] + fn a_newer_format_is_newer_whatever_its_features_look_like() { + let mut f = JournalFormat::default(); + for newer in [ + json!({"format": 2, "features": {"actors": true}}), + json!({"format": 2, "features": [1, 2]}), + json!({"format": 2}), + json!({"format": 4_294_967_296_u64, "features": []}), + ] { + assert!( + is_newer(&f.admit(&line(FORMAT_SET, newer.clone()))), + "{newer}" + ); + } + assert_eq!(f, JournalFormat::default()); + // The same shapes under a format this version reads are damage. + for bad in [ + json!({"format": 1, "features": {"actors": true}}), + json!({"format": 1, "features": [1, 2]}), + ] { + assert!(matches!( + f.admit(&line(FORMAT_SET, bad)), + Err(StoreError::JournalCorrupt(_)) + )); + } + } +} diff --git a/crates/ledger-store/src/journal.rs b/crates/ledger-store/src/journal.rs index 09dc247..0a33f99 100644 --- a/crates/ledger-store/src/journal.rs +++ b/crates/ledger-store/src/journal.rs @@ -10,11 +10,12 @@ //! and the chain verifies. use crate::error::StoreError; +use crate::format::JournalFormat; use serde::{Deserialize, Serialize}; use serde_json::Value; use sha2::{Digest, Sha256}; use std::fs::{File, OpenOptions}; -use std::io::{BufRead, BufReader, Write}; +use std::io::Write; use std::path::{Path, PathBuf}; /// `prev` value of the first journal line: 64 zeros. @@ -28,9 +29,11 @@ pub struct JournalLine { /// UTC timestamp, RFC 3339. Recorded, never trusted for ordering. pub ts: String, /// Event type, e.g. `account.added`, `entry.posted`, `asset.tagged`: - /// every type the store writes has a replay arm in - /// `replay_line_into_ledger` (an unknown type is refused as corrupt), - /// and journal text in the viewer (`ledger-gui` tests check both). + /// every type the store writes is in [`crate::format::EVENT_TYPES`], + /// has a replay arm in `replay_line_into_ledger`, and has journal text + /// in the viewer (`ledger-store` and `ledger-gui` tests check). A type + /// this version doesn't know is refused as `NewerJournal`, not as + /// corrupt: a newer wanderbooks wrote it. #[serde(rename = "type")] pub line_type: String, /// The full domain facts. Everything a replay needs. @@ -67,6 +70,12 @@ fn sort_value(v: &Value) -> Value { } } +/// The keys of a journal line, the same in every journal format +/// (rules.md B-27): a newer format adds its fields inside `payload`, never +/// beside it, and a line's hash is always the SHA-256 of its canonical +/// JSON ([`canonical_json`]), whatever the format. +pub const ENVELOPE_KEYS: [&str; 5] = ["payload", "prev", "seq", "ts", "type"]; + impl JournalLine { /// Parse one journal line, refusing what the journal never holds. /// @@ -74,22 +83,70 @@ impl JournalLine { /// GOAL: read a journal line so that its canonical form (which the /// hash chain covers) is a fixed point. /// SOURCE: R-2 (no floats in money paths); found by fuzz_journal_line - /// (2026-10-13): a float payload re-canonicalized differently. + /// (2026-10-13): a float payload re-canonicalized differently; + /// rules.md B-27 (the envelope is frozen). /// INVARIANTS: /// 1. NEVER accepts a floating-point number anywhere in the line: /// amounts are decimal strings and ids are integers. - /// 2. For an accepted line, parse(canonical) re-canonicalizes to the + /// 2. NEVER drops a key: a line with a key beside the five of + /// ENVELOPE_KEYS is refused (the store's reader calls it newer). + /// 3. For an accepted line, parse(canonical) re-canonicalizes to the /// same bytes. /// ``` pub fn parse(raw: &[u8]) -> Result { - let value: Value = serde_json::from_slice(raw).map_err(|e| e.to_string())?; - if has_float(&value) { - return Err("a number with a fraction or exponent; the journal holds none".into()); + let read = read_line(raw)?; + if read.unknown_keys.is_empty() { + Ok(read.line) + } else { + Err(format!( + "line fields this version doesn't know: {}", + read.unknown_keys.join(", ") + )) } - serde_json::from_value(value).map_err(|e| e.to_string()) } } +/// One journal line as the store's readers take it in. +pub(crate) struct ReadLine { + /// The line, without any key this version doesn't know. + pub line: JournalLine, + /// SHA-256 of the line's canonical JSON, every key included: what the + /// next line's `prev` must be. + pub hash: String, + /// Keys beside [`ENVELOPE_KEYS`], sorted: none in any line this + /// version writes. + pub unknown_keys: Vec, +} + +/// Read one journal line (JournalLine::parse INVARIANT 1), keeping the +/// keys this version doesn't know out of the line but in its hash, so a +/// reader can still check the chain through it. +pub(crate) fn read_line(raw: &[u8]) -> Result { + let mut value: Value = serde_json::from_slice(raw).map_err(|e| e.to_string())?; + if has_float(&value) { + return Err("a number with a fraction or exponent; the journal holds none".into()); + } + let hash = sha256_hex(canonical_json(&value).as_bytes()); + let mut unknown_keys = Vec::new(); + if let Value::Object(map) = &mut value { + unknown_keys = map + .keys() + .filter(|k| !ENVELOPE_KEYS.contains(&k.as_str())) + .cloned() + .collect(); + unknown_keys.sort(); + for k in &unknown_keys { + map.remove(k); + } + } + let line = serde_json::from_value(value).map_err(|e| e.to_string())?; + Ok(ReadLine { + line, + hash, + unknown_keys, + }) +} + fn has_float(v: &Value) -> bool { match v { Value::Number(n) => n.is_f64(), @@ -193,16 +250,72 @@ impl JournalWriter { /// Read and fully verify a journal: every line parses, seqs are gapless from /// 1, and the hash chain is intact. Any failure names the offending seq. +/// This is the journal's own integrity, whatever its lines' types; opening +/// the books also passes every line through the format gate +/// ([`read_journal_and_format`]). pub fn read_and_verify_journal(path: &Path) -> Result, StoreError> { - let file = File::open(path)?; - let reader = BufReader::new(file); + verify_lines(&std::fs::read(path)?, |_| Ok(())) +} + +/// [`read_and_verify_journal`] with the format gate, also returning the +/// journal format in force. +/// +/// ```text +/// GOAL: books read whole and verified, or refused with the reason. +/// SOURCE: docs/receipt-store.md "Line format"; rules.md B-27. +/// INVARIANTS: +/// 1. Line by line, in order: parse, seq, chain, then the format gate. +/// 2. Damage before the first line this version can't read is +/// JournalCorrupt, that line included: its seq and prev are checked +/// before the gate, and its own hash by the next line's prev +/// (format::settle), so a flipped bit in its type is corrupt, not +/// newer. +/// 3. At the first whole, chained line this version can't read (an +/// unknown type, a newer journal.format_set, or a key beside the +/// envelope's): SHOULD stop with NewerJournal (verified when the next +/// line's prev matches it; unverified, saying "newer or damaged", +/// when there is no next line or it has no prev this version can +/// read) +/// BECAUSE a newer format may frame later lines in ways this version +/// can't check; judging them is the newer version's job. +/// 4. The journal is read as bytes, split on newlines, as the viewer +/// reads it (crate::Snapshot): a line that isn't UTF-8 JSON is +/// JournalCorrupt, never an I/O error. +/// ``` +pub fn read_journal_and_format( + path: &Path, +) -> Result<(Vec, JournalFormat), StoreError> { + verify_journal_and_format(&std::fs::read(path)?) +} + +/// [`read_journal_and_format`] over a journal's bytes (`raw`). +pub(crate) fn verify_journal_and_format( + raw: &[u8], +) -> Result<(Vec, JournalFormat), StoreError> { + let mut format = JournalFormat::default(); + let lines = verify_lines(raw, |line| format.admit(line))?; + Ok((lines, format)) +} + +/// Check each line of `raw` (bytes, split on newlines; a final newline +/// ends the last line): its parse, seq and chain, then `admit` on it, in +/// order; the first failure stops the read. +fn verify_lines( + raw: &[u8], + mut admit: impl FnMut(&JournalLine) -> Result<(), StoreError>, +) -> Result, StoreError> { let mut lines = Vec::new(); + if raw.is_empty() { + return Ok(lines); + } + let body = raw.strip_suffix(b"\n").unwrap_or(raw); let mut expected_prev = GENESIS_PREV.to_string(); - for (i, raw) in reader.lines().enumerate() { - let raw = raw?; - let line: JournalLine = JournalLine::parse(raw.as_bytes()).map_err(|e| { + let mut raws = body.split(|&b| b == b'\n').enumerate().peekable(); + while let Some((i, raw)) = raws.next() { + let read = read_line(raw).map_err(|e| { StoreError::JournalCorrupt(format!("seq {}: unparseable line: {e}", i + 1)) })?; + let line = &read.line; let want_seq = i as u64 + 1; if line.seq != want_seq { return Err(StoreError::JournalCorrupt(format!( @@ -216,8 +329,14 @@ pub fn read_and_verify_journal(path: &Path) -> Result, StoreErr line.seq ))); } - expected_prev = sha256_hex(line.to_canonical().as_bytes()); - lines.push(line); + if let Err(refused) = crate::format::gate(&read, &mut admit) { + // A line's own hash is checked only by the next line's prev: + // read it before saying whether the books are newer or damaged. + let next = raws.peek().map(|(_, next)| *next); + return Err(crate::format::settle(refused, &read.hash, next)); + } + expected_prev = read.hash; + lines.push(read.line); } Ok(lines) } diff --git a/crates/ledger-store/src/lib.rs b/crates/ledger-store/src/lib.rs index 1bd8781..cb1e09f 100644 --- a/crates/ledger-store/src/lib.rs +++ b/crates/ledger-store/src/lib.rs @@ -20,6 +20,7 @@ pub mod blob; pub mod error; +pub mod format; pub mod journal; pub mod reconcilable; pub mod schema; @@ -29,6 +30,7 @@ pub mod viewer; pub use blob::BlobStore; pub use error::StoreError; +pub use format::{JournalFormat, SUPPORTED_JOURNAL_FORMAT}; pub use journal::{GENESIS_PREV, JournalLine, JournalWriter}; pub use snapshot::{DiskCheck, Snapshot}; pub use store::{ diff --git a/crates/ledger-store/src/snapshot.rs b/crates/ledger-store/src/snapshot.rs index 38317df..c10700a 100644 --- a/crates/ledger-store/src/snapshot.rs +++ b/crates/ledger-store/src/snapshot.rs @@ -16,7 +16,8 @@ use crate::blob::BlobStore; use crate::error::StoreError; -use crate::journal::{GENESIS_PREV, JournalLine, sha256_hex}; +use crate::format::JournalFormat; +use crate::journal::{GENESIS_PREV, JournalLine}; use crate::store::{ Attachment, BLOBS_DIR, DB_FILE, JOURNAL_FILE, Reconciliation, Statement, Store, Subject, apply_line_to_sqlite, compare_projections, projection_rows, query_attachment_links, @@ -71,6 +72,8 @@ pub struct Snapshot { offset: u64, /// Hash of the last line read: what the next line must chain onto. head: String, + /// The journal format in force (the format gate's record). + format: JournalFormat, } impl Snapshot { @@ -91,6 +94,7 @@ impl Snapshot { lines: Vec::new(), offset: 0, head: GENESIS_PREV.to_string(), + format: JournalFormat::default(), }; snap.read_new()?; Ok(snap) @@ -114,7 +118,16 @@ impl Snapshot { /// 1. IF nothing was appended: SHOULD return 0 and change nothing. /// 2. IF the file shrank or no longer chains (a restore, a repaired /// tail): SHOULD reload from scratch. - /// 3. IF a complete new line fails its hash or parse: SHOULD refuse. + /// 3. IF a complete new line fails its hash or parse (a blank line + /// included, as the writer reads it): SHOULD refuse. + /// 4. IF a complete new line is whole and chained but one this + /// version can't read (an unknown type, a newer format, a key + /// beside the envelope's): SHOULD refuse with NewerJournal + /// (verified or not as format::settle finds it: a next line whose + /// prev doesn't match makes it corrupt), applying + /// none of the new lines (as the writer refuses the books whole), + /// and refuse the same way on every refresh until wanderbooks is + /// updated; what was read before stays. /// ``` pub fn refresh(&mut self) -> Result { let len = std::fs::metadata(self.data_dir.join(JOURNAL_FILE))?.len(); @@ -145,40 +158,63 @@ impl Snapshot { return Ok(0); }; let complete = &buf[..=last_newline]; - let mut added = 0; - for raw in complete.split(|&b| b == b'\n').filter(|l| !l.is_empty()) { - let line: JournalLine = JournalLine::parse(raw).map_err(|e| { - StoreError::JournalCorrupt(format!( - "seq {}: unparseable line: {e}", - self.lines.len() + 1 - )) + // First check every new line, as the writer does when it opens the + // books: parse, seq, chain, then the format gate, stopping at the + // first failure. A batch holding a line this version can't read is + // refused whole, so the viewer and the writer give the same answer + // for the same journal. + let mut format = self.format.clone(); + let mut head = self.head.clone(); + let mut checked: Vec<(JournalLine, String, u64)> = Vec::new(); + let mut raws = complete + .split_inclusive(|&b| b == b'\n') + .map(|raw| (raw.len() as u64, &raw[..raw.len() - 1])) + .peekable(); + while let Some((len, raw)) = raws.next() { + let seq = self.lines.len() + checked.len() + 1; + // A blank complete line is no line at all: corrupt, as the + // writer finds it (it never writes one). + let read = crate::journal::read_line(raw).map_err(|e| { + StoreError::JournalCorrupt(format!("seq {seq}: unparseable line: {e}")) })?; - let want_seq = self.lines.len() as u64 + 1; - if line.seq != want_seq { + let line = &read.line; + if line.seq != seq as u64 { return Err(StoreError::JournalCorrupt(format!( - "seq gap: expected {want_seq}, found {}", + "seq gap: expected {seq}, found {}", line.seq ))); } - if line.prev != self.head { + if line.prev != head { return Err(StoreError::JournalCorrupt(format!( "hash chain broken at seq {}", line.seq ))); } + if let Err(refused) = crate::format::gate(&read, &mut |l| format.admit(l)) { + // Its own hash is checked by the next line's prev, when + // there is one (format::settle). + let next = raws.peek().map(|(_, raw)| *raw); + return Err(crate::format::settle(refused, &read.hash, next)); + } + head = read.hash; + checked.push((read.line, head.clone(), len)); + } + // Then apply them. The offset advances line by line, so if replay + // refuses one, the snapshot holds exactly the lines before it. + let added = checked.len(); + for (line, hash, len) in checked { if line.seq == 1 { - let code = line.payload["base_currency"].as_str().unwrap_or("USD"); - self.base_currency = Currency::new(code) - .map_err(|_| StoreError::JournalCorrupt(format!("bad base_currency {code}")))?; + // The writer's rule: the first line is a sound store.created. + self.base_currency = crate::store::created_currency(&line)?; self.ledger = Ledger::new(self.base_currency); } + self.format.admit(&line)?; replay_line_into_ledger(&mut self.ledger, &line, self.base_currency)?; apply_line_to_sqlite(&self.conn, &line)?; - self.head = sha256_hex(line.to_canonical().as_bytes()); + self.head = hash; self.lines.push(line); - added += 1; + self.offset += len; } - self.offset += complete.len() as u64; Ok(added) } @@ -192,6 +228,12 @@ impl Snapshot { self.base_currency } + /// The journal format in force: format 1 unless the books declared + /// another with `journal.format_set`. + pub fn journal_format(&self) -> &JournalFormat { + &self.format + } + /// The data directory. pub fn data_dir(&self) -> &Path { &self.data_dir diff --git a/crates/ledger-store/src/store.rs b/crates/ledger-store/src/store.rs index 0892829..3c7ad0f 100644 --- a/crates/ledger-store/src/store.rs +++ b/crates/ledger-store/src/store.rs @@ -32,6 +32,7 @@ use crate::blob::BlobStore; use crate::error::StoreError; +use crate::format::JournalFormat; use crate::journal::{ GENESIS_PREV, JournalLine, JournalWriter, read_and_verify_journal, sha256_hex, }; @@ -202,6 +203,8 @@ pub struct Store { blobs: BlobStore, ledger: Ledger, base_currency: Currency, + /// The journal format in force (the format gate's record). + format: JournalFormat, } /// One filing of bytes into the blob store and journal. @@ -254,6 +257,7 @@ impl Store { blobs: BlobStore::open(&data_dir.join(BLOBS_DIR))?, ledger: Ledger::new(base_currency), base_currency, + format: JournalFormat::default(), }; let line = store.journal.append( "store.created", @@ -279,10 +283,19 @@ impl Store { /// SCENARIOS: /// 1. IF there is no journal: SHOULD refuse (NotFound). /// 2. IF another writer has the books: SHOULD refuse (Locked). - /// 3. IF the last line is cut off (no newline): SHOULD set it aside, - /// truncate, and journal `journal.tail_repaired`. + /// 3. IF the last line is cut off (no newline) and every complete + /// line passes (its chain, the format gate, the first line's + /// store.created, and its replay into the ledger, all in memory): + /// SHOULD set it aside, truncate, and journal + /// `journal.tail_repaired`; IF a complete line fails: SHOULD + /// refuse with the journal byte for byte as it was. /// 4. IF a complete line fails its hash, parse, or `seq`: SHOULD refuse /// - NEVER repair or skip it (that is corruption, for a human). + /// 4a. IF a whole, chained line is one this version can't read (an + /// unknown type, a newer journal format, a key beside the + /// envelope's): SHOULD refuse with NewerJournal before touching + /// the database or the journal, a cut-off last line included + /// BECAUSE a newer wanderbooks wrote it, and may be appending. /// 5. IF the database is missing: SHOULD rebuild it from the journal. /// 6. IF the database fails SQLite's integrity check, or records a /// base currency other than the journal's: SHOULD keep it aside @@ -297,14 +310,31 @@ impl Store { } // The writer lock first: nothing is read until no one else writes. let lock = lock_books(data_dir)?; - // An append cut off mid-line was never acknowledged: set it aside. - let torn = crate::journal::repair_torn_tail(&journal_path)?; - // Single verified scan; the writer reuses it. - let lines = read_and_verify_journal(&journal_path)?; - let journal = JournalWriter::from_lines(&journal_path, &lines)?; + // Single verified scan of the complete lines, through the format + // gate, before anything is changed: books from a newer wanderbooks + // are refused whole, never read in part, and never altered (a + // newer version may be mid-append). The writer reuses the scan. + let raw = std::fs::read(&journal_path)?; + let complete = raw.iter().rposition(|&b| b == b'\n').map_or(0, |i| i + 1); + let (lines, format) = crate::journal::verify_journal_and_format(&raw[..complete])?; // The base currency comes from the journal, the source of truth; a // journal without a sound store.created line is corrupt. let base_currency = journal_currency(&lines)?; + // Every complete line replayed, in memory, before anything on disk + // changes (SCENARIO 3): a line the books can't apply refuses the + // open with the journal and the database as they were. + let mut ledger = Ledger::new(base_currency); + for line in &lines { + replay_line_into_ledger(&mut ledger, line, base_currency)?; + } + // Only then is an append cut off mid-line (never acknowledged) set + // aside, and journaled below. + let torn = if complete < raw.len() { + crate::journal::repair_torn_tail(&journal_path)? + } else { + None + }; + let journal = JournalWriter::from_lines(&journal_path, &lines)?; let db_path = data_dir.join(DB_FILE); let mut fresh_db = !db_path.exists(); let damage = if fresh_db { @@ -350,8 +380,9 @@ impl Store { journal, conn, blobs: BlobStore::open(&data_dir.join(BLOBS_DIR))?, - ledger: Ledger::new(base_currency), + ledger, base_currency, + format, }; store.catch_up(&lines)?; if let Some(t) = torn { @@ -448,8 +479,9 @@ impl Store { Ok(()) } - /// Replay the journal: hydrate the in-memory ledger fully, and apply to - /// SQLite every line past `last_applied_seq`. + /// Catch the projection up: apply to SQLite every line past + /// `last_applied_seq` (the ledger is already the replay of every line, + /// done in memory by [`Store::open`] before anything changed). fn catch_up(&mut self, lines: &[JournalLine]) -> Result<(), StoreError> { let last_applied: u64 = self .conn @@ -461,11 +493,10 @@ impl Store { .optional()? .and_then(|s| s.parse().ok()) .unwrap_or(0); - for line in lines { - replay_line_into_ledger(&mut self.ledger, line, self.base_currency)?; - if line.seq > last_applied { - apply_line_to_sqlite(&self.conn, line)?; - } + // The ledger is already their replay (Store::open): only the + // projection is caught up here. + for line in lines.iter().filter(|l| l.seq > last_applied) { + apply_line_to_sqlite(&self.conn, line)?; } Ok(()) } @@ -484,6 +515,41 @@ impl Store { self.base_currency } + /// The journal format in force: format 1 unless the books declared + /// another with `journal.format_set`. + pub fn journal_format(&self) -> &JournalFormat { + &self.format + } + + /// Declare the journal format: journal a `journal.format_set` line. + /// A writer declares the format before the first event that needs it, + /// so an older reader stops at the declaration and says it is too old. + /// + /// ```text + /// GOAL: books that tell every reader which format they are in. + /// SOURCE: rules.md B-27; plan/served.md 5.1 (format gate). + /// SCENARIOS: + /// 1. IF the format is newer than SUPPORTED_JOURNAL_FORMAT, or 0: + /// SHOULD refuse (FormatRefused), journaling nothing + /// BECAUSE this version couldn't open the books it left. + /// 2. IF the format is lower than the one in force: SHOULD refuse + /// BECAUSE events written under it may need the higher one. + /// 3. IF a feature isn't in KNOWN_FEATURES: SHOULD refuse. + /// 4. Otherwise: SHOULD journal it (features sorted, once each) and + /// make it the format in force. + /// ``` + pub fn declare_journal_format( + &mut self, + format: u32, + features: &[&str], + ) -> Result<(), StoreError> { + let payload = self.format.declaration(format, features)?; + let line = self.append_journal(crate::format::FORMAT_SET, payload)?; + self.format = crate::format::declared(&line)?; + apply_line_to_sqlite(&self.conn, &line)?; + Ok(()) + } + /// Every torn journal tail set aside on open, oldest first, as /// (when, bytes, file it was saved to). pub fn tail_repairs(&self) -> Result, StoreError> { @@ -4416,11 +4482,14 @@ pub(crate) fn replay_line_into_ledger( )?; } "journal.tail_repaired" => {} - other => { - return Err(StoreError::JournalCorrupt(format!( - "unknown line type: {other}" - ))); + "journal.format_set" => { + // No ledger state; a format this version can't read is + // refused here too, for callers that skip the gate. + crate::format::declared(line)?; } + // A whole, chained line of a type this version doesn't know: a + // newer wanderbooks wrote it (the format gate). + _ => return Err(crate::format::unknown_type(line)), } Ok(()) } @@ -4643,7 +4712,7 @@ pub(crate) fn apply_line_to_sqlite( fn apply_line_rows(conn: &Connection, line: &JournalLine) -> Result<(), StoreError> { let p = &line.payload; match line.line_type.as_str() { - "store.created" | "journal.tail_repaired" => {} + "store.created" | "journal.tail_repaired" | "journal.format_set" => {} "account.added" => { conn.execute( "INSERT INTO accounts(id, code, name, type, parent, placeholder, currency) VALUES (?,?,?,?,?,?,?)", @@ -5197,11 +5266,7 @@ fn apply_line_rows(conn: &Connection, line: &JournalLine) -> Result<(), StoreErr ], )?; } - other => { - return Err(StoreError::JournalCorrupt(format!( - "unknown line type: {other}" - ))); - } + _ => return Err(crate::format::unknown_type(line)), } conn.execute( "INSERT INTO events(type, payload, created_at) VALUES (?,?,?)", @@ -5607,12 +5672,20 @@ fn row_to_attachment(r: &rusqlite::Row) -> rusqlite::Result { }) } -/// The base currency of the books, from the journal's first line, which -/// must be `store.created`. +/// The base currency of a journal: its first line's, which must be a +/// sound `store.created` ([`created_currency`]). fn journal_currency(lines: &[JournalLine]) -> Result { let first = lines .first() .ok_or_else(|| StoreError::JournalCorrupt("journal has no store.created line".into()))?; + created_currency(first) +} + +/// The base currency the journal's first line sets: it must be a +/// `store.created` line naming a currency, or the journal is corrupt. The +/// one rule for the writer ([`Store::open`]) and the viewer +/// ([`crate::Snapshot`]), so the two never read different books. +pub(crate) fn created_currency(first: &JournalLine) -> Result { if first.line_type != "store.created" { return Err(StoreError::JournalCorrupt( "journal's first line is not store.created".into(), diff --git a/crates/ledger-store/tests/format.rs b/crates/ledger-store/tests/format.rs new file mode 100644 index 0000000..c7a4c17 --- /dev/null +++ b/crates/ledger-store/tests/format.rs @@ -0,0 +1,863 @@ +//! The format gate: books written by a newer wanderbooks are refused as +//! newer, never as corrupt, and never read in part. +//! +//! ```text +//! GOAL: an older wanderbooks tells the human to update, not that the +//! books are damaged, and never shows books that differ from the +//! record without saying so. +//! SOURCE: rules.md B-27; plan/served.md 5.1 (format gate). +//! INVARIANTS: +//! 1. A whole, chained line of a type this version doesn't know, or a +//! journal.format_set newer than it reads, is NewerJournal: from +//! Store::open, Snapshot::load and Snapshot::refresh alike. +//! 2. Damage (a bad hash, a bad seq, a line that isn't JSON) stays +//! JournalCorrupt, whatever the damaged line's type. +//! 3. A declared format this version reads replays like any other line. +//! ``` + +use chrono::NaiveDate; +use ledger_core::{AccountType, Currency, Money, Side}; +use ledger_store::format::{EVENT_TYPES, FORMAT_SET, KNOWN_FEATURES}; +use ledger_store::journal::{canonical_json, sha256_hex}; +use ledger_store::{ + DB_FILE, JOURNAL_FILE, JournalFormat, JournalWriter, SUPPORTED_JOURNAL_FORMAT, Snapshot, Store, + StoreError, +}; +use rust_decimal::Decimal; +use serde_json::json; +use std::path::{Path, PathBuf}; + +fn test_dir(name: &str) -> PathBuf { + let dir = + std::env::temp_dir().join(format!("wanderbooks-format-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +/// Books with one posted entry, closed again. +fn books(dir: &Path) { + let mut store = Store::create(dir, Currency::USD).unwrap(); + let cash = store + .add_account("1000", "Cash", AccountType::Asset, None, false) + .unwrap(); + let equity = store + .add_account("3000", "Capital", AccountType::Equity, None, false) + .unwrap(); + let e = store + .create_entry(NaiveDate::from_ymd_opt(2026, 10, 1).unwrap(), "Capital") + .unwrap(); + let m = Money::new(Decimal::from(500), Currency::USD); + store.add_line(e, cash, Side::Debit, m).unwrap(); + store.add_line(e, equity, Side::Credit, m).unwrap(); + store.post_entry(e).unwrap(); +} + +/// Append whole, correctly chained lines, as a newer wanderbooks would, +/// returning their seqs. +fn append(dir: &Path, lines: &[(&str, serde_json::Value)]) -> Vec { + let mut w = JournalWriter::open(&dir.join(JOURNAL_FILE)).unwrap(); + lines + .iter() + .map(|(t, p)| w.append(t, p.clone()).unwrap().seq) + .collect() +} + +/// NewerJournal for line `want_seq`. `want_verified`: the next line +/// chains onto it, so that line is intact (and the message vouches for +/// nothing after it); otherwise the message says newer or damaged, never +/// "intact". +fn assert_newer( + got: Result, + want_seq: u64, + want_type: &str, + want_need: &str, + want_verified: bool, +) { + match got { + Err(e @ StoreError::NewerJournal { .. }) => { + let text = e.to_string(); + let StoreError::NewerJournal { + seq, + event_type, + need, + verified, + } = e + else { + unreachable!() + }; + assert_eq!(seq, want_seq); + assert_eq!(event_type, want_type); + assert_eq!(need, want_need); + assert_eq!(verified, want_verified, "{text}"); + assert!(text.contains("newer version of wanderbooks"), "{text}"); + assert!(text.contains("Update wanderbooks"), "{text}"); + assert!(!text.contains("corrupt"), "{text}"); + assert!(!text.contains("not damaged"), "{text}"); + if verified { + assert!(text.contains(&format!("Line {seq} is intact")), "{text}"); + assert!(text.contains("can't check the lines after it"), "{text}"); + } else { + assert!(!text.contains("intact"), "{text}"); + assert!(text.contains("or the line is damaged"), "{text}"); + assert!(text.contains("run verify"), "{text}"); + } + } + Err(e) => panic!("expected NewerJournal, got {e:?}"), + Ok(_) => panic!("expected NewerJournal, the books opened"), + } +} + +fn assert_corrupt(got: Result) { + match got { + Err(StoreError::JournalCorrupt(_)) => {} + Err(e) => panic!("expected JournalCorrupt, got {e:?}"), + Ok(_) => panic!("expected JournalCorrupt, the books opened"), + } +} + +/// Oracle (plan/served.md 5.1): an older reader meets books a newer +/// wanderbooks wrote. A valid line of an unknown type, and a declared +/// format 2, are each NewerJournal, from the writer and the viewer alike, +/// and the refusal reads nothing further: the database isn't rebuilt and +/// the journal is untouched. +#[test] +fn old_reader_gets_newer_journal() { + let cases = [ + ( + "settings.set", + json!({"key": "company.name", "value": "X"}), + "an event type this version doesn't know", + ), + ( + FORMAT_SET, + json!({"format": 2, "features": []}), + "journal format 2", + ), + ( + FORMAT_SET, + json!({"format": 1, "features": ["actors"]}), + "journal features actors", + ), + ]; + for (i, (line_type, payload, need)) in cases.into_iter().enumerate() { + let dir = test_dir(&format!("newer-{i}")); + books(&dir); + // A later line of a known type changes nothing: the first line + // this version can't read is the one named. + let seq = append( + &dir, + &[ + (line_type, payload), + ( + "fx.rate_set", + json!({"currency": "EUR", "date": "2026-10-01", "rate": "1.10"}), + ), + ], + )[0]; + std::fs::remove_file(dir.join(DB_FILE)).unwrap(); + let journal = std::fs::read(dir.join(JOURNAL_FILE)).unwrap(); + + assert_newer(Store::open(&dir), seq, line_type, need, true); + assert!( + !dir.join(DB_FILE).exists(), + "a refused open rebuilt the database" + ); + assert_eq!(std::fs::read(dir.join(JOURNAL_FILE)).unwrap(), journal); + assert_newer(Snapshot::load(&dir), seq, line_type, need, true); + } +} + +/// A viewer already following the books meets a newer line mid-stream: +/// refresh says NewerJournal (every time, until it is updated), applies +/// none of the lines written with it (as the writer refuses the books +/// whole), and keeps showing the books as it last read them. +#[test] +fn a_viewer_following_the_books_gets_newer_journal() { + let dir = test_dir("follow"); + books(&dir); + let mut snap = Snapshot::load(&dir).unwrap(); + let before = ledger_core::reports::snapshot(snap.ledger()).unwrap(); + let declared = append(&dir, &[(FORMAT_SET, json!({"format": 1, "features": []}))])[0]; + assert_eq!(snap.refresh().unwrap(), 1); + let seq = append( + &dir, + &[ + ( + "fx.rate_set", + json!({"currency": "EUR", "date": "2026-10-01", "rate": "1.10"}), + ), + ("entry.evidence_inherited", json!({"entry_id": 1})), + ], + )[1]; + for _ in 0..2 { + assert_newer( + snap.refresh(), + seq, + "entry.evidence_inherited", + "an event type this version doesn't know", + false, + ); + assert_eq!(snap.journal_seq(), declared); + assert_eq!( + ledger_core::reports::snapshot(snap.ledger()).unwrap(), + before + ); + } +} + +/// The writer and the viewer refuse the same journal the same way, even +/// when an earlier line's facts are bad too: every line passes the gate +/// before any is replayed. +#[test] +fn the_writer_and_the_viewer_agree_on_newer_books() { + let dir = test_dir("agree"); + books(&dir); + let seq = append( + &dir, + &[ + // Chained and of a known type, but replay refuses it. + ("entry.posted", json!({"id": 99})), + ("settings.set", json!({"key": "company.name"})), + ], + )[1]; + let need = "an event type this version doesn't know"; + assert_newer(Store::open(&dir), seq, "settings.set", need, false); + assert_newer(Snapshot::load(&dir), seq, "settings.set", need, false); +} + +/// Damage stays corruption, whatever the damaged line's type: the gate +/// sees only lines that parsed, chained and kept their seq, so damage up +/// to and including the first line this version can't read is +/// JournalCorrupt. (Lines after that one are the newer version's to judge: +/// its format may frame them differently.) +#[test] +fn damaged_journals_are_still_corrupt() { + type Damage = fn(&mut Vec); + let damages: [(&str, Damage); 4] = [ + ("hash", |lines| { + let last = lines.last_mut().unwrap(); + *last = last.replace("\"prev\":\"", "\"prev\":\"0"); + }), + ("seq", |lines| { + let n = lines.len(); + let last = lines.last_mut().unwrap(); + *last = last.replace(&format!("\"seq\":{n}"), &format!("\"seq\":{}", n + 1)); + }), + ("json", |lines| { + let last = lines.last_mut().unwrap(); + last.truncate(last.len() / 2); + }), + ("payload", |lines| { + // The line before, edited after it was written: the last + // line's prev no longer matches. + let n = lines.len(); + lines[n - 2] = lines[n - 2].replace("\"Capital\"", "\"Capitol\""); + }), + ]; + let newer = [ + (FORMAT_SET, json!({"format": 2, "features": []})), + ("settings.set", json!({"key": "company.name"})), + ]; + for (name, damage) in damages { + for (i, last) in newer.iter().enumerate() { + let dir = test_dir(&format!("damaged-{name}-{i}")); + books(&dir); + append(&dir, std::slice::from_ref(last)); + let path = dir.join(JOURNAL_FILE); + let mut lines: Vec = std::fs::read_to_string(&path) + .unwrap() + .lines() + .map(str::to_string) + .collect(); + let whole = lines.clone(); + damage(&mut lines); + assert_ne!(lines, whole, "{name} damaged nothing"); + std::fs::write(&path, lines.join("\n") + "\n").unwrap(); + assert_corrupt(Store::open(&dir)); + assert_corrupt(Snapshot::load(&dir)); + } + } +} + +/// A declared format this version reads opens, verifies, rebuilds, and +/// reads the same through the viewer; the format in force is recorded. +#[test] +fn a_declared_format_1_replays() { + let dir = test_dir("format-1"); + books(&dir); + let mut store = Store::open(&dir).unwrap(); + assert_eq!(store.journal_format(), &JournalFormat::default()); + let before = ledger_core::reports::snapshot(store.ledger()).unwrap(); + store.declare_journal_format(1, &[]).unwrap(); + assert_eq!(store.journal_format().format, 1); + store.verify().unwrap(); + drop(store); + + let mut store = Store::open(&dir).unwrap(); + store.verify().unwrap(); + store.rebuild().unwrap(); + store.verify().unwrap(); + assert_eq!(store.journal_format(), &JournalFormat::default()); + assert_eq!( + ledger_core::reports::snapshot(store.ledger()).unwrap(), + before + ); + let snap = Snapshot::load(&dir).unwrap(); + assert_eq!(snap.journal_format(), &JournalFormat::default()); + assert_eq!(snap.journal().last().unwrap().line_type, FORMAT_SET); + assert_eq!( + ledger_core::reports::snapshot(snap.ledger()).unwrap(), + before + ); +} + +/// A writer never declares a format it couldn't read back: refused, and +/// the journal is byte-identical. +#[test] +fn a_format_this_version_cannot_read_is_never_declared() { + let dir = test_dir("declare-refused"); + books(&dir); + let mut store = Store::open(&dir).unwrap(); + let journal = std::fs::read(dir.join(JOURNAL_FILE)).unwrap(); + let unknown = ["no-such-feature"]; + assert!(!KNOWN_FEATURES.contains(&unknown[0])); + for (format, features) in [ + (SUPPORTED_JOURNAL_FORMAT + 1, &[][..]), + (0, &[][..]), + (1, &unknown[..]), + ] { + assert!( + matches!( + store.declare_journal_format(format, features), + Err(StoreError::FormatRefused(_)) + ), + "declared {format} {features:?}" + ); + assert_eq!(std::fs::read(dir.join(JOURNAL_FILE)).unwrap(), journal); + } + assert_eq!(store.journal_format(), &JournalFormat::default()); +} + +/// Every type the store dispatches on is one the gate admits: a type with +/// a replay or projection arm in store.rs that the list lacked would be +/// refused as newer in the books that hold it. (The converse, every listed +/// type has both arms, is checked inside the crate.) +#[test] +fn every_event_the_store_replays_passes_the_gate() { + let src = include_str!("../src/store.rs"); + let mut arms: Vec<&str> = src + .lines() + .map(str::trim_start) + .map(|l| l.strip_prefix("| ").unwrap_or(l)) + .filter_map(|l| l.strip_prefix('"')) + .filter_map(|l| l.split_once('"')) + .filter(|(_, rest)| { + let rest = rest.trim_start(); + rest.starts_with("=>") || rest.starts_with('|') || rest.is_empty() + }) + .map(|(t, _)| t) + .filter(|t| { + t.split_once('.').is_some_and(|(a, b)| { + !a.is_empty() + && !b.is_empty() + && t.chars().all(|c| { + c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_' || c == '.' + }) + }) + }) + .collect(); + arms.sort_unstable(); + arms.dedup(); + assert!(arms.len() > 50, "found only {arms:?}"); + for t in &arms { + assert!( + EVENT_TYPES.contains(t), + "{t} is replayed but not in EVENT_TYPES" + ); + } + for t in EVENT_TYPES { + assert!( + arms.contains(t), + "{t} is in EVENT_TYPES but has no arm in store.rs" + ); + } +} + +/// Rewrite the journal's lines with `edit`, as damage on disk would. +fn edit_journal(dir: &Path, edit: impl FnOnce(&mut Vec)) { + let path = dir.join(JOURNAL_FILE); + let mut lines: Vec = std::fs::read_to_string(&path) + .unwrap() + .lines() + .map(str::to_string) + .collect(); + let whole = lines.clone(); + edit(&mut lines); + assert_ne!(lines, whole, "the edit changed nothing"); + std::fs::write(&path, lines.join("\n") + "\n").unwrap(); +} + +/// A line's own hash is checked only by the next line's prev, so the +/// gate's refusal waits for it (rules.md B-27): a flipped bit in a middle +/// line's type, or a declared format edited from 1 to 2, reads like books +/// from a newer wanderbooks until the next line shows the line changed. +/// That is corruption, from the writer and the viewer alike. +#[test] +fn a_damaged_line_that_looks_newer_is_corrupt_when_the_next_line_checks_it() { + type Edit = fn(&mut Vec); + let edits: [(&str, Edit); 2] = [ + ("type", |lines| { + // account.added (seq 2) becomes a type no version writes. + lines[1] = lines[1].replace("\"account.added\"", "\"account.addee\""); + }), + ("format", |lines| { + let at = lines.iter().position(|l| l.contains(FORMAT_SET)).unwrap(); + lines[at] = lines[at].replace("\"format\":1", "\"format\":2"); + }), + ]; + for (name, edit) in edits { + let dir = test_dir(&format!("flipped-{name}")); + books(&dir); + let mut store = Store::open(&dir).unwrap(); + store.declare_journal_format(1, &[]).unwrap(); + store + .add_account("1010", "Savings", AccountType::Asset, None, false) + .unwrap(); + drop(store); + edit_journal(&dir, edit); + assert_corrupt(Store::open(&dir)); + assert_corrupt(Snapshot::load(&dir)); + } +} + +/// When the line the gate refuses is the journal's last, nothing checks +/// its bytes: a damaged type and a genuinely newer one look the same. The +/// refusal says exactly that (newer or damaged; update, then verify), +/// never that the books are not damaged. +#[test] +fn an_unreadable_last_line_is_reported_as_newer_or_damaged() { + let dir = test_dir("flipped-last"); + books(&dir); + let seq = append( + &dir, + &[( + "fx.rate_set", + json!({"currency": "EUR", "date": "2026-10-01", "rate": "1.10"}), + )], + )[0]; + edit_journal(&dir, |lines| { + let last = lines.last_mut().unwrap(); + *last = last.replace("\"fx.rate_set\"", "\"fx.rate_sex\""); + }); + let need = "an event type this version doesn't know"; + assert_newer(Store::open(&dir), seq, "fx.rate_sex", need, false); + assert_newer(Snapshot::load(&dir), seq, "fx.rate_sex", need, false); +} + +/// A genuinely newer line with a whole, chained line after it: its bytes +/// are checked, so the books are newer and that line is intact. +#[test] +fn a_newer_line_the_next_line_chains_onto_is_newer_and_intact() { + let dir = test_dir("newer-verified"); + books(&dir); + let seq = append( + &dir, + &[ + ("entry.evidence_inherited", json!({"entry_id": 1})), + ("entry.evidence_inherited", json!({"entry_id": 1})), + ], + )[0]; + let need = "an event type this version doesn't know"; + assert_newer( + Store::open(&dir), + seq, + "entry.evidence_inherited", + need, + true, + ); + assert_newer( + Snapshot::load(&dir), + seq, + "entry.evidence_inherited", + need, + true, + ); +} + +/// A line the gate refuses, followed by a complete line this version +/// can't even parse as JSON (or a blank one), is unverified: nothing +/// checks the refused line, so it is newer or damaged, never corrupt for +/// certain (a newer format may frame its lines its own way; rules.md +/// B-27), and never intact. +#[test] +fn a_refused_line_followed_by_a_garbled_line_is_unverified() { + for (i, garbled) in ["{not json", "", "[1,2]", "{\"prev\":7}"] + .iter() + .enumerate() + { + let dir = test_dir(&format!("newer-then-garbled-{i}")); + books(&dir); + let seq = append( + &dir, + &[("entry.evidence_inherited", json!({"entry_id": 1}))], + )[0]; + edit_journal(&dir, |lines| lines.push(garbled.to_string())); + let need = "an event type this version doesn't know"; + let t = "entry.evidence_inherited"; + assert_newer(Store::open(&dir), seq, t, need, false); + assert_newer(Snapshot::load(&dir), seq, t, need, false); + } +} + +/// The journal's last line, as raw bytes, and its hash: what the next +/// line's prev must be. +fn last_line(dir: &Path) -> (u64, String) { + let raw = std::fs::read(dir.join(JOURNAL_FILE)).unwrap(); + let body = raw.strip_suffix(b"\n").unwrap(); + let last = body.rsplit(|&b| b == b'\n').next().unwrap(); + let v: serde_json::Value = serde_json::from_slice(last).unwrap(); + (v["seq"].as_u64().unwrap(), sha256_hex(last)) +} + +/// Append one line's bytes and a newline, as another writer would. +fn append_raw(dir: &Path, bytes: &[u8]) { + let mut f = std::fs::OpenOptions::new() + .append(true) + .open(dir.join(JOURNAL_FILE)) + .unwrap(); + std::io::Write::write_all(&mut f, bytes).unwrap(); +} + +/// A newer format frames the lines after its declaration its own way +/// (rules.md B-27): a float, a string seq, a key of its own. This version +/// reads only the next line's prev, so it never calls such books corrupt: +/// the declaration is intact when that prev checks it, unverified when the +/// next line has no prev it can read, and corrupt only when a prev it can +/// read doesn't match. The writer and the viewer agree. +#[test] +fn a_newer_format_frames_its_lines_its_own_way() { + type Next = fn(u64, &str) -> serde_json::Value; + let nexts: [(&str, Next, Option); 6] = [ + ( + "float", + |seq, prev| json!({"seq": seq, "ts": "t", "type": "settings.set", "payload": {"rate": 0.5}, "prev": prev}), + Some(true), + ), + ( + "string-seq", + |seq, prev| json!({"seq": seq.to_string(), "ts": "t", "type": "settings.set", "payload": {}, "prev": prev}), + Some(true), + ), + ( + "envelope-key", + |seq, prev| json!({"seq": seq, "ts": "t", "type": "settings.set", "payload": {}, "prev": prev, "actor": "agent"}), + Some(true), + ), + ( + "no-prev", + |seq, prev| json!({"seq": seq, "ts": "t", "type": "settings.set", "payload": {}, "parent": prev}), + Some(false), + ), + ( + "prev-not-text", + |seq, _| json!({"seq": seq, "ts": "t", "type": "settings.set", "payload": {}, "prev": 1.5}), + Some(false), + ), + ( + "wrong-prev", + |seq, _| json!({"seq": seq, "ts": "t", "type": "settings.set", "payload": {"rate": 0.5}, "prev": "00"}), + None, + ), + ]; + for (name, next, verified) in nexts { + let dir = test_dir(&format!("v2-{name}")); + books(&dir); + let seq = append( + &dir, + &[(FORMAT_SET, json!({"format": 2, "features": ["actors"]}))], + )[0]; + let (_, hash) = last_line(&dir); + append_raw( + &dir, + format!("{}\n", canonical_json(&next(seq + 1, &hash))).as_bytes(), + ); + match verified { + Some(v) => { + assert_newer(Store::open(&dir), seq, FORMAT_SET, "journal format 2", v); + assert_newer(Snapshot::load(&dir), seq, FORMAT_SET, "journal format 2", v); + } + None => { + assert_corrupt(Store::open(&dir)); + assert_corrupt(Snapshot::load(&dir)); + } + } + } +} + +/// A key beside the envelope's is never dropped (rules.md B-27): the line +/// is refused as newer, and its hash covers the key, so the next line +/// still checks it (intact), a next line hashed without the key shows it +/// was added (corrupt), and as the last line it is unverified. The writer +/// and the viewer agree, and JournalLine::parse refuses such a line. +#[test] +fn an_unknown_envelope_key_is_newer_and_checked_through() { + for case in ["chained", "hashed-without-the-key", "last"] { + let dir = test_dir(&format!("envelope-{case}")); + books(&dir); + let (seq, prev) = last_line(&dir); + let seq = seq + 1; + let line = json!({"seq": seq, "ts": "2026-10-16T00:00:00Z", "type": "fx.rate_set", + "payload": {"currency": "EUR", "date": "2026-10-01", "rate": "1.10"}, + "prev": prev, "actor": "agent"}); + let raw = canonical_json(&line); + let err = ledger_store::JournalLine::parse(raw.as_bytes()).unwrap_err(); + assert!(err.contains("actor"), "{err}"); + append_raw(&dir, format!("{raw}\n").as_bytes()); + let without = { + let mut l = line.clone(); + l.as_object_mut().unwrap().remove("actor"); + sha256_hex(canonical_json(&l).as_bytes()) + }; + let next_prev = match case { + "chained" => Some(sha256_hex(raw.as_bytes())), + "hashed-without-the-key" => Some(without), + _ => None, + }; + if let Some(p) = &next_prev { + let next = json!({"seq": seq + 1, "ts": "t", "type": "fx.rate_set", + "payload": {"currency": "EUR", "date": "2026-10-02", "rate": "1.10"}, "prev": p}); + append_raw(&dir, format!("{}\n", canonical_json(&next)).as_bytes()); + } + let need = "journal line fields this version doesn't know (actor)"; + match case { + "hashed-without-the-key" => { + assert_corrupt(Store::open(&dir)); + assert_corrupt(Snapshot::load(&dir)); + } + _ => { + let v = case == "chained"; + assert_newer(Store::open(&dir), seq, "fx.rate_set", need, v); + assert_newer(Snapshot::load(&dir), seq, "fx.rate_set", need, v); + } + } + } +} + +/// A line that isn't UTF-8 is damage, for the writer as for the viewer +/// (they read the journal the same way: bytes, split on newlines), never +/// an I/O error. +#[test] +fn a_line_that_is_not_utf8_is_corrupt_for_the_viewer_and_the_writer() { + for at in ["end", "after-a-newer-line"] { + let dir = test_dir(&format!("not-utf8-{at}")); + books(&dir); + let path = dir.join(JOURNAL_FILE); + if at == "end" { + append_raw(&dir, b"{\"seq\":99,\"x\":\"\xff\xfe\"}\n"); + assert_corrupt(Store::open(&dir)); + assert_corrupt(Snapshot::load(&dir)); + assert_corrupt(ledger_store::journal::read_and_verify_journal(&path)); + } else { + // After a refused line, it is a next line without a readable + // prev: unverified, for both. + let seq = append(&dir, &[("settings.set", json!({}))])[0]; + append_raw(&dir, b"{\"prev\":\"\xff\"}\n"); + let need = "an event type this version doesn't know"; + assert_newer(Store::open(&dir), seq, "settings.set", need, false); + assert_newer(Snapshot::load(&dir), seq, "settings.set", need, false); + } + } +} + +/// An older version never changes books it can't read: a newer line +/// followed by a cut-off last line (a newer version mid-append) is +/// refused as newer, and the journal is byte for byte as it was, with +/// nothing set aside. Books this version reads still get the repair. +#[test] +fn an_old_reader_never_repairs_newer_books() { + let dir = test_dir("newer-torn"); + books(&dir); + let seq = append(&dir, &[("settings.set", json!({"key": "company.name"}))])[0]; + append_raw(&dir, b"{\"seq\":"); + let path = dir.join(JOURNAL_FILE); + let before = std::fs::read(&path).unwrap(); + let need = "an event type this version doesn't know"; + assert_newer(Store::open(&dir), seq, "settings.set", need, false); + assert_eq!(std::fs::read(&path).unwrap(), before); + let set_aside: Vec<_> = std::fs::read_dir(&dir) + .unwrap() + .filter_map(Result::ok) + .filter(|e| e.file_name().to_string_lossy().contains(".torn.")) + .collect(); + assert!(set_aside.is_empty(), "{set_aside:?}"); + + // The same cut-off tail after lines this version reads is repaired, + // and the repair is journaled. + let dir = test_dir("known-torn"); + books(&dir); + append_raw(&dir, b"{\"seq\":"); + let store = Store::open(&dir).unwrap(); + assert_eq!(store.tail_repairs().unwrap().len(), 1); +} + +/// B-27 ("a cut-off last line is set aside only when the lines before it +/// pass"; Store::open SCENARIO 3): a complete line that chains and passes +/// the gate but can't be applied (a post of an entry that doesn't exist), +/// or a first line that isn't store.created, refuses the open with the +/// journal byte for byte as it was and nothing set aside, whatever comes +/// after it: a cut-off tail stays where it is, for a human to look at. +#[test] +fn a_bad_complete_line_before_a_torn_tail_leaves_the_journal_as_it_was() { + let unapplied = test_dir("bad-then-torn"); + books(&unapplied); + append(&unapplied, &[("entry.posted", json!({"entry_id": 999}))]); + let headless = test_dir("headless-then-torn"); + append( + &headless, + &[( + "account.added", + json!({"id": 1, "code": "1000", "name": "Cash", "account_type": "Asset"}), + )], + ); + for dir in [&unapplied, &headless] { + append_raw(dir, b"{\"seq\":"); + let path = dir.join(JOURNAL_FILE); + let before = std::fs::read(&path).unwrap(); + let had_db = dir.join(ledger_store::DB_FILE).exists(); + assert!(Store::open(dir).is_err(), "{}", dir.display()); + assert_eq!(std::fs::read(&path).unwrap(), before, "{}", dir.display()); + let set_aside: Vec<_> = std::fs::read_dir(dir) + .unwrap() + .filter_map(Result::ok) + .filter(|e| e.file_name().to_string_lossy().contains(".torn.")) + .collect(); + assert!(set_aside.is_empty(), "{set_aside:?}"); + assert_eq!( + dir.join(ledger_store::DB_FILE).exists(), + had_db, + "no database made for books that can't open" + ); + } + assert!(matches!( + Store::open(&headless), + Err(StoreError::JournalCorrupt(_)) + )); +} + +/// The viewer applies the writer's rule for the first line (B-27: the +/// two read the same journal the same way): it must be a store.created +/// naming a currency. Any other first line, or a store.created without a +/// sound base_currency, is corrupt for both; neither guesses a currency. +#[test] +fn the_first_line_is_store_created_for_the_viewer_and_the_writer() { + let firsts = [ + ( + "account.added", + json!({"id": 1, "code": "1000", "name": "Cash", "account_type": "Asset"}), + ), + ("store.created", json!({})), + ("store.created", json!({"base_currency": 5})), + ("store.created", json!({"base_currency": "dollars"})), + ("journal.tail_repaired", json!({"bytes": 1, "sha256": "00"})), + ]; + for (n, (t, payload)) in firsts.into_iter().enumerate() { + let dir = test_dir(&format!("first-line-{n}")); + append(&dir, &[(t, payload.clone())]); + assert_corrupt(Store::open(&dir)); + assert_corrupt(Snapshot::load(&dir)); + } + // A sound one opens for both, in its own currency. + let dir = test_dir("first-line-ok"); + append(&dir, &[("store.created", json!({"base_currency": "EUR"}))]); + assert_eq!(Store::open(&dir).unwrap().base_currency(), Currency::EUR); + assert_eq!(Snapshot::load(&dir).unwrap().base_currency(), Currency::EUR); +} + +/// A blank complete line is damage, for the viewer as for the writer: the +/// writer never writes one, and the two read the same journal the same +/// way (S9 b, 2026-10-16). +#[test] +fn a_blank_line_is_corrupt_for_the_viewer_and_the_writer() { + for at in ["middle", "end"] { + let dir = test_dir(&format!("blank-{at}")); + books(&dir); + edit_journal(&dir, |lines| match at { + "middle" => lines.insert(2, String::new()), + _ => lines.push(String::new()), + }); + assert_corrupt(Store::open(&dir)); + assert_corrupt(Snapshot::load(&dir)); + } + // A viewer following the books meets one appended later. + let dir = test_dir("blank-follow"); + books(&dir); + let mut snap = Snapshot::load(&dir).unwrap(); + let mut f = std::fs::OpenOptions::new() + .append(true) + .open(dir.join(JOURNAL_FILE)) + .unwrap(); + std::io::Write::write_all(&mut f, b"\n").unwrap(); + assert_corrupt(snap.refresh()); +} + +/// The writer and the viewer, opening and following, say the same thing, +/// word for word, about the same damaged line (the newline that ends it +/// is no part of the line): a JSON value cut off where its line ends. +#[test] +fn the_viewer_and_the_writer_say_the_same_of_a_damaged_line() { + for (tag, raw) in [ + ("string", &b"{\"seq\":9,\"note\":\"cut"[..]), + ("object", &b"{\"seq\":9,"[..]), + ] { + let dir = test_dir(&format!("same-words-{tag}")); + books(&dir); + let mut snap = Snapshot::load(&dir).unwrap(); + append_raw(&dir, &[raw, b"\n"].concat()); + let writer = Store::open(&dir).err().expect("damaged books opened"); + assert!(matches!(writer, StoreError::JournalCorrupt(_)), "{writer}"); + let writer = writer.to_string(); + assert!(writer.contains("unparseable line"), "{writer}"); + let viewer = Snapshot::load(&dir).err().expect("damaged books loaded"); + assert_eq!(viewer.to_string(), writer, "{tag}: opening"); + let following = snap.refresh().expect_err("damaged books followed"); + assert_eq!(following.to_string(), writer, "{tag}: following"); + } +} + +/// read_journal_and_format reads every line, in order, as the plain +/// verified read does, with the format in force. +#[test] +fn read_journal_and_format_gives_every_line_and_the_format() { + let dir = test_dir("read-and-format"); + books(&dir); + append( + &dir, + &[( + FORMAT_SET, + json!({"format": SUPPORTED_JOURNAL_FORMAT, "features": []}), + )], + ); + let path = dir.join(JOURNAL_FILE); + let (lines, format) = ledger_store::journal::read_journal_and_format(&path).unwrap(); + let want: Vec<(u64, String)> = ledger_store::journal::read_and_verify_journal(&path) + .unwrap() + .into_iter() + .map(|l| (l.seq, l.line_type)) + .collect(); + assert!(want.len() > 5, "{want:?}"); + assert_eq!( + lines + .into_iter() + .map(|l| (l.seq, l.line_type)) + .collect::>(), + want + ); + assert_eq!( + format, + JournalFormat { + format: SUPPORTED_JOURNAL_FORMAT, + features: vec![] + } + ); +} diff --git a/crates/ledger-store/tests/store.rs b/crates/ledger-store/tests/store.rs index 96fe205..ef8a860 100644 --- a/crates/ledger-store/tests/store.rs +++ b/crates/ledger-store/tests/store.rs @@ -1523,7 +1523,8 @@ mod refusals { /// of books written before them. A journal holding what the new rules /// refuse (a child account of another type, partially overlapping periods, /// a draft line on a missing account) must still open and verify; replay -/// goes through Ledger::replay_*. +/// goes through Ledger::replay_*. Each phase adds what its books hold (P0, +/// 2026-10-16: the journal format declared, `journal.format_set` format 1). #[test] fn books_written_under_older_rules_still_open() { use serde_json::json; @@ -1535,6 +1536,8 @@ fn books_written_under_older_rules_still_open() { drop(store); // What an older wanderbooks could have written. let mut w = JournalWriter::open(&dir.join(ledger_store::JOURNAL_FILE)).unwrap(); + w.append("journal.format_set", json!({"format": 1, "features": []})) + .unwrap(); w.append( "account.added", json!({"id": 2, "code": "1010", "name": "Rent", "account_type": "Expense", @@ -1573,6 +1576,7 @@ fn books_written_under_older_rules_still_open() { let _ = std::fs::remove_file(dir.join(ledger_store::DB_FILE)); let store = Store::open(&dir).expect("older books still open"); store.verify().unwrap(); + assert_eq!(store.journal_format().format, 1); assert_eq!(store.ledger().chart().len(), 2); assert_eq!(store.ledger().list_periods().len(), 2); let lines = store.ledger().get_entry(1).unwrap().lines(); diff --git a/docs/spec/rules.md b/docs/spec/rules.md index 445d084..874c19f 100644 --- a/docs/spec/rules.md +++ b/docs/spec/rules.md @@ -603,3 +603,69 @@ against the books). `tests/spec_parties.rs`, `tests/spec_vault.rs` (every response, the log, and every file in the books dir outside `vault/` scanned for the TINs in any formatting). + +## B-27 The journal format gate: newer is not corrupt + +**Status:** DECISION (Stephen, 2026-10-16; `plan/served.md` 5.1 and P0); +DECISION (P0 review, 2026-10-16: a line's own hash is checked before the +line is called newer; the envelope and its hash are frozen across +formats) + +Every journal line has a type, and a version of wanderbooks reads a fixed +set of them (`ledger_store::format::EVENT_TYPES`). A newer version may +write a type an older one doesn't know, or declare a newer journal format +with a `journal.format_set {format, features}` line before the first event +that needs it. An older reader stops at the first line it can't read and +says so: it never skips a line (the books it showed would differ from the +record without saying so), and it never calls newer books corrupt. + +What every journal format shares is frozen, so that an older reader can +always follow the chain one line past the first line it can't read. A +line is the envelope `{seq, ts, type, payload, prev}` and nothing beside +it; its hash (what the next line's `prev` holds) is the SHA-256 of its +canonical JSON: keys sorted at every depth, no whitespace. A newer format +puts its new fields inside `payload`, never in the envelope, and never +changes how a line is hashed. A line with a key beside the envelope's is +refused as newer, never read with the key dropped, and its hash covers +the key, so the next line still checks it. + +Damage stays corruption. A line's seq and `prev` are checked before the +gate, and its own hash is checked by the next line's `prev`; so when the +gate refuses a line, the reader reads the next one first, taking only its +`prev` (any JSON object with a string `prev`: it may use floats, a string +seq or keys of a format this version doesn't read). If that `prev` doesn't +match, the refused line was altered after it was written (a flipped bit in +its type reads like an unknown type): corrupt. If it matches, the refused +line is intact; the reader says so, and that it can't check the lines +after it. If there is no next line, or it has no `prev` this version can +read, nothing checks the refused line: the reader says it needs a newer +wanderbooks or the line is damaged, and to run verify after updating, and +never that it is intact. A newer format number is newer whatever the rest +of its declaration looks like (a newer format may shape its features +differently); a malformed declaration of a format this version reads is +corrupt. A blank line, or one that isn't UTF-8 JSON, is damage, to the +writer and the viewer alike: both read the journal as bytes split on +newlines. A writer never declares a format it couldn't read back, or +lowers the format in force. Opening the books checks every complete line +before it changes anything: a cut-off last line is set aside only when the +lines before it pass, so an older version never alters newer books. + +**Enforced:** `ledger_store::format` (`JournalFormat::admit`, `declared`, +`gate`, `settle`, `declaration`), `journal::ENVELOPE_KEYS`, +`JournalLine::parse`, `journal::read_journal_and_format`, +`Snapshot::read_new`, `Store::open`, `Store::declare_journal_format`, +`StoreError::NewerJournal` (its `verified` flag picks the message). +**Checked:** `ledger-store` `tests/format.rs` (`old_reader_gets_newer_journal`, +`a_damaged_line_that_looks_newer_is_corrupt_when_the_next_line_checks_it`, +`an_unreadable_last_line_is_reported_as_newer_or_damaged`, +`a_newer_line_the_next_line_chains_onto_is_newer_and_intact`, +`a_newer_format_frames_its_lines_its_own_way`, +`an_unknown_envelope_key_is_newer_and_checked_through`, +`a_refused_line_followed_by_a_garbled_line_is_unverified`, +`a_line_that_is_not_utf8_is_corrupt_for_the_viewer_and_the_writer`, +`an_old_reader_never_repairs_newer_books`, +`a_blank_line_is_corrupt_for_the_viewer_and_the_writer`, +`damaged_journals_are_still_corrupt`) and the unit tests in `format.rs`; +`ledger-gui` `tests/newer_journal.rs`; `ledger-mcp` `tests/startup.rs`; +the `fuzz_store_replay` target (the writer and the viewer agree on newer +versus bad). diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index ac45566..eff98b9 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -1421,6 +1421,7 @@ dependencies = [ "ledger-core", "rust_decimal", "serde", + "serde_json", "toml 1.1.8+spec-1.1.0", ] @@ -1458,6 +1459,7 @@ dependencies = [ "chrono", "csv", "ledger-core", + "qrcode", "rust_decimal", "rust_xlsxwriter", "serde", @@ -1959,6 +1961,12 @@ version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" +[[package]] +name = "qrcode" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d68782463e408eb1e668cf6152704bd856c78c5b6417adaee3203d8f4c1fc9ec" + [[package]] name = "quick-error" version = "2.0.1" diff --git a/fuzz/fuzz_targets/fuzz_store_replay.rs b/fuzz/fuzz_targets/fuzz_store_replay.rs index a5d7a9a..820c466 100644 --- a/fuzz/fuzz_targets/fuzz_store_replay.rs +++ b/fuzz/fuzz_targets/fuzz_store_replay.rs @@ -3,13 +3,18 @@ // Raw bytes (fuzz_store_open) almost never pass the hash chain, so they // never reach replay. Here the journal is written by the real // JournalWriter (whole lines, gapless seq, correct prev), starting with a -// real store.created; the events after it are fuzz-chosen types with -// payloads built from the field names replay reads. Oracles: +// real store.created most of the time (else a store.created with a +// fuzz-chosen currency or payload, or another type: the writer and the +// viewer share one rule for the first line); the events after it are +// fuzz-chosen types with payloads built from the field names replay +// reads. Oracles: // 1. Opening never panics or hangs (an Err is fine: bad facts are // refused). // 2. Books that open must verify (live ledger = replay = projection). // 3. The viewer (Snapshot::load) accepts exactly the journals the writer // does, and then holds the same ledger. +// 4. When both refuse, they agree on whether the books are newer +// (NewerJournal: a type or format this version doesn't read) or bad. #![no_main] use arbitrary::{Arbitrary, Unstructured}; @@ -25,6 +30,7 @@ const TYPES: &[&str] = &[ "line.cleared", "payroll.run", "period.closed", "period.created", "receipt.attached", "receipt.linked", "receipt.unlinked", "recurring.generated", "recurring.registered", "schedule.posted", "schedule.registered", "vendor.1099_set", "vendor.added", + "journal.format_set", "settings.set", ]; const FIELDS: &[&str] = &[ @@ -38,7 +44,7 @@ const FIELDS: &[&str] = &[ "rate", "reason", "recurring_id", "reversal_of", "reverses", "salvage_value", "schedule_id", "share", "side", "size_bytes", "start", "tax_category", "tin_on_file", "total", "useful_life_months", "vendor_id", "type", "account_type", "rounding", - "members", "distributions", "sweep", + "members", "distributions", "sweep", "format", "features", ]; /// A value replay might plausibly see: small ids, amounts as decimal @@ -78,7 +84,13 @@ fuzz_target!(|data: &[u8]| { std::fs::create_dir_all(&dir).unwrap(); { let mut w = ledger_store::JournalWriter::open(&dir.join(ledger_store::JOURNAL_FILE)).unwrap(); - w.append("store.created", json!({"base_currency": "USD"})).unwrap(); + let (first, payload) = match u.int_in_range(0..=7).unwrap_or(0) { + 0 => ("store.created", object(&mut u, 0).unwrap_or_default()), + 1 => (*u.choose(TYPES).unwrap_or(&"account.added"), object(&mut u, 0).unwrap_or_default()), + 2 => ("store.created", json!({"base_currency": value(&mut u, 0).unwrap_or_default()})), + _ => ("store.created", json!({"base_currency": "USD"})), + }; + w.append(first, payload).unwrap(); for _ in 0..u.int_in_range(0..=24).unwrap_or(0) { let (Ok(t), Ok(p)) = (u.choose(TYPES), object(&mut u, 0)) else { break }; w.append(t, p).unwrap(); @@ -94,7 +106,12 @@ fuzz_target!(|data: &[u8]| { "the viewer and the writer read different books" ); } - (Err(_), Err(_)) => {} + (Err(a), Err(b)) => { + let newer = |e: &ledger_store::StoreError| { + matches!(e, ledger_store::StoreError::NewerJournal { .. }) + }; + assert_eq!(newer(&a), newer(&b), "the writer says {a}; the viewer says {b}"); + } (Ok(_), Err(e)) => panic!("the writer opens books the viewer refuses: {e}"), (Err(e), Ok(_)) => panic!("the viewer loads books the writer refuses: {e}"), }