swaits/wanderbooks
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

P3: the books' format moves only with the owner's go-ahead (D-39)

Browse Source 
Books an update could write a newer format for open on their old format
and are asked at every open; anything needing a new feature is refused
until the owner agrees with the passphrase, after a verified backup that
includes the keys. New books start on the current format. Each projection
schema has its own database file, so an older version still opens the
books until the owner agrees.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01T3gXMVt7XJPi6ens1or9Dj
This commit is contained in:
swaitsandClaude Opus 5.5 committed 2026-10-21 14:04:11 -06:00
1 parent b0a9fd8ad4
commit 3a9aa92315
62 files changed
+5309 -628
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

crates/ledger-store/src/backup.rs
+137
View File
@@ -966,6 +966,90 @@ pub fn drill(
checked
}
/// Prove the book's own archive holds the book in `book_dir` exactly as it
/// is now, its journal's head `head` (INVARIANT 3, and more): the archive
/// cloned into `scratch` (a new folder, removed after) and proven whole as
/// a restore proves a backup, its head `head`, its journal the book's byte
/// for byte, and its TIN vault and keys the book's, file for file and byte
/// for byte. What the format upgrade asks of the backup it makes first
/// (plan/served.md 3.9: "a verified local backup that includes the
/// keys"). The caller holds the store's lock, so the book can't move
/// while it is compared.
pub fn prove_archive(
git: &Git,
book_dir: &Path,
head: &(u64, String),
scratch: &Path,
) -> Result<Proven, BackupError> {
let archive = book_dir.join(ARCHIVE_DIR);
let source = archive.display().to_string();
let proven = verify(git, &source, scratch).and_then(|v| {
if v.head != *head {
return Err(BackupError::Damaged(format!(
"it holds the books through event {}, and they are at event {}",
v.head.0, head.0
)));
}
if std::fs::read(book_dir.join(crate::JOURNAL_FILE))?
!= std::fs::read(v.tree.join(crate::JOURNAL_FILE))?
{
return Err(BackupError::Damaged(
"its journal isn't the books' own, byte for byte".into(),
));
}
for kept in [crate::root::VAULT_DIR, crate::root::KEYS_DIR] {
if held_files(book_dir, kept)? != held_files(&v.tree, kept)? {
return Err(BackupError::Damaged(format!(
"its {} isn't the books' own, file for file",
if kept == crate::root::KEYS_DIR {
"keys folder"
} else {
"TIN vault"
}
)));
}
}
let git_dir = format!("--git-dir={}", v.tree.join(".git").display());
let commit = git.run("rev-parse", &[os(&git_dir), os("rev-parse"), os("HEAD")])?;
Ok(Proven {
head: v.head,
blobs: v.blobs,
commit,
})
});
let _ = std::fs::remove_dir_all(scratch);
proven
}
/// The book's archive proven to hold the book as it is ([`prove_archive`]).
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Proven {
/// The journal's head it holds: the book's.
pub head: (u64, String),
/// Stored files re-hashed.
pub blobs: u64,
/// The commit proven: the archive's latest.
pub commit: String,
}
/// Every file under `rel` in `base`, by its path there, with its bytes
/// (none when `rel` isn't there).
fn held_files(base: &Path, rel: &str) -> Result<Vec<(String, Vec<u8>)>, BackupError> {
if std::fs::symlink_metadata(base.join(rel)).is_err() {
return Ok(Vec::new());
}
let mut names = Vec::new();
files_under(base, Path::new(rel), &mut names)?;
names.sort();
names
.into_iter()
.map(|n| {
let bytes = std::fs::read(base.join(&n))?;
Ok((n, bytes))
})
.collect()
}
/// Whether the journal at `backup` is a byte prefix of the one at `live`
/// (a drill: the backup holds the book's history, nothing else).
pub fn is_prefix(live: &Path, backup: &Path) -> std::io::Result<bool> {
@@ -1471,6 +1555,59 @@ mod tests {
);
}
/// The backup a format upgrade makes first (plan 3.9): the archive
/// proves to hold the book exactly as it is, its keys and vault file
/// for file; a key, a vault record or the journal that differs from
/// the archive's latest, or a head it doesn't hold, is refused; and
/// the scratch folder goes either way.
#[test]
fn the_archive_proves_to_hold_the_book_as_it_is_keys_and_all() {
let git = Git::find().expect("git is installed");
let dir = tempfile::tempdir().unwrap();
let b = dir.path().join("book");
let mut s = book(&b);
let keys = b.join(crate::root::KEYS_DIR);
std::fs::create_dir(&keys).unwrap();
std::fs::write(keys.join(crate::root::LEGACY_KEY_FILE), b"ab".repeat(32)).unwrap();
let vault = b.join(crate::root::VAULT_DIR);
std::fs::create_dir(&vault).unwrap();
std::fs::write(vault.join("tin-1.bin"), b"sealed").unwrap();
let made = commit(&git, &b, "acme", s.head()).unwrap().unwrap();
let scratch = dir.path().join("check");
let proven = prove_archive(&git, &b, &s.head(), &scratch).unwrap();
assert_eq!(proven.head, s.head());
assert_eq!(proven.commit, made.commit);
assert_eq!(proven.blobs, 1);
assert!(!scratch.exists(), "the scratch folder is removed");
let refused = |b: &Path, head: &(u64, String)| {
let e = prove_archive(&git, b, head, &scratch).unwrap_err();
assert!(!scratch.exists());
e.to_string()
};
// A key that isn't the one backed up.
std::fs::write(keys.join(crate::root::LEGACY_KEY_FILE), b"cd".repeat(32)).unwrap();
assert!(refused(&b, &s.head()).contains("keys folder"));
std::fs::write(keys.join(crate::root::LEGACY_KEY_FILE), b"ab".repeat(32)).unwrap();
// A key the backup lacks.
std::fs::write(keys.join("other.key"), b"x").unwrap();
assert!(refused(&b, &s.head()).contains("keys folder"));
std::fs::remove_file(keys.join("other.key")).unwrap();
// A vault record that isn't the one backed up.
std::fs::write(vault.join("tin-1.bin"), b"sealed again").unwrap();
assert!(refused(&b, &s.head()).contains("TIN vault"));
std::fs::write(vault.join("tin-1.bin"), b"sealed").unwrap();
assert!(prove_archive(&git, &b, &s.head(), &scratch).is_ok());
// The books moved on since the backup.
s.add_account("6000", "Fees", AccountType::Expense, None, false)
.unwrap();
let e = refused(&b, &s.head());
assert!(e.contains("they are at event"), "{e}");
// The head it holds, but the journal on disk changed under it.
let old_head = made.head.clone();
let e = refused(&b, &old_head);
assert!(e.contains("byte for byte"), "{e}");
}
#[test]
fn a_backup_holding_a_link_or_a_stranger_is_refused_before_checkout() {
let git = Git::find().expect("git is installed");
crates/ledger-store/src/error.rs
+30
View File
@@ -82,6 +82,18 @@ pub enum StoreError {
#[error("{0}")]
FormatRefused(String),
/// A write that needs a journal format or feature these books haven't
/// declared: they were written by an older version, and wait for the
/// owner's go-ahead to move to this version's format (the format
/// upgrade: plan/served.md 3.9, D-39). Nothing was written; everything
/// that needs nothing new still works.
#[error("{}", format_upgrade_text(need))]
FormatUpgradePending {
/// What the write needs: "journal format 2", "the journal
/// feature x".
need: String,
},
/// Replay produced different ids than the journal recorded.
#[error("replay mismatch: {0}")]
ReplayMismatch(String),
@@ -137,6 +149,24 @@ pub enum StoreError {
CurrencyMismatch(String, String),
}
/// The guard code a refusal for books waiting for their format upgrade
/// carries (plan/served.md 3.9: `format_upgrade_pending`).
pub const FORMAT_UPGRADE_PENDING: &str = "format_upgrade_pending";
/// What [`StoreError::FormatUpgradePending`] says: what the write needs,
/// that nothing was changed, that only the owner can give the go-ahead
/// (and where), and that the rest still works.
fn format_upgrade_text(need: &str) -> String {
format!(
"{FORMAT_UPGRADE_PENDING}: this needs {need}, and these books are still in the journal \
format an older version of wanderbooks wrote: they are waiting for the owner's \
go-ahead to upgrade to this version's format. Nothing was changed. Only the owner can \
give it, with the owner passphrase: in the app (it asks when it opens the books) or \
with `wanderbooks books upgrade ID` at a terminal; no tool can. Until then everything \
that needs nothing new works as before"
)
}
/// What [`StoreError::NewerJournal`] says. Only a line whose own hash was
/// checked lets it say that line is intact; it never vouches for the
/// lines after it, which this version can't read.
crates/ledger-store/src/format.rs
+266 -15
View File
@@ -20,10 +20,16 @@
//! can't read, and say whether that line is intact.
//!
//! A line an earlier version can't read is announced by a feature
//! ([`KNOWN_FEATURES`]): the writer declares it (`journal.format_set`,
//! the features in force and the new one) just before the first line that
//! needs it, so the earlier reader stops at the declaration, and books
//! that never use the feature stay readable by it.
//! ([`KNOWN_FEATURES`]): the books declare it (`journal.format_set`)
//! before the first line that needs it, so the earlier reader stops at the
//! declaration.
//!
//! The other way round, books an older version wrote are in an older
//! format: this version reads them as they are, and writes nothing that
//! needs a format or feature they haven't declared until the owner says
//! go (the format upgrade, rules.md B-27, plan/served.md 3.9, D-39:
//! [`JournalFormat::upgrade`], [`current`]). New books declare this
//! version's whole format as they are created.
use crate::error::StoreError;
use crate::journal::{JournalLine, ReadLine};
@@ -35,11 +41,11 @@ pub const SUPPORTED_JOURNAL_FORMAT: u32 = 2;
/// Optional journal features this version reads. Each marks lines an
/// older version can't read: a new event type (which it would refuse as
/// unknown), or a field or value on an event type it knows (which it
/// would misread silently). A writer declares the feature
/// (`journal.format_set`) before its first such line
/// ([`crate::Store`]'s `ensure_feature`), so the older version stops
/// there instead (rules.md B-27). Books that never use one stay readable
/// by versions before it.
/// would misread silently). The books declare it (`journal.format_set`)
/// before their first such line, so the older version stops there instead
/// (rules.md B-27). Every one is declared at once: by new books as they
/// are created, and by older books at their format upgrade ([`current`]);
/// until then a write that needs one is refused.
pub const KNOWN_FEATURES: &[&str] = &[
MEMBERS_CUMULATIVE,
FEATURE_ACCOUNT_RENAMED,
@@ -59,16 +65,15 @@ pub const KNOWN_FEATURES: &[&str] = &[
/// rule, which can split it differently.
pub const MEMBERS_CUMULATIVE: &str = "members_cumulative";
/// The feature declared before the first `account.renamed` (P3, defect
/// 15).
/// Feature: `account.renamed`, a new event type (P3, defect 15).
pub const FEATURE_ACCOUNT_RENAMED: &str = "account_renamed";
/// The feature declared before the first `settings.set` naming
/// `policy.require_tax_category` (P3, defect 8).
/// Feature: a `settings.set` naming `policy.require_tax_category`, a key
/// an earlier version doesn't know (P3, defect 8).
pub const FEATURE_REQUIRE_TAX_CATEGORY: &str = "require_tax_category";
/// The feature a `settings.set` naming `key` needs declared first: a key
/// an earlier version doesn't know.
/// The feature a `settings.set` naming `key` needs the books to have
/// declared: a key an earlier version doesn't know.
pub(crate) fn setting_feature(key: &str) -> Option<&'static str> {
match key {
"policy.require_tax_category" => Some(FEATURE_REQUIRE_TAX_CATEGORY),
@@ -262,6 +267,122 @@ impl JournalFormat {
}
}
/// The journal format this version writes: the newest format it reads,
/// with every feature it knows, sorted. New books declare it as they are
/// created ([`crate::Store::create`]), and books in an older format move
/// to it in one step at their format upgrade
/// ([`crate::Store::upgrade_format`]).
pub fn current() -> JournalFormat {
let mut features: Vec<String> = KNOWN_FEATURES.iter().map(|f| f.to_string()).collect();
features.sort_unstable();
features.dedup();
JournalFormat {
format: SUPPORTED_JOURNAL_FORMAT,
features,
}
}
/// A format upgrade books wait for: from the format they are in to this
/// version's ([`current`]).
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct FormatUpgrade {
/// The format the books are in.
pub from: JournalFormat,
/// The format the upgrade declares: this version's.
pub to: JournalFormat,
}
impl FormatUpgrade {
/// What the upgrade adds, in words: a newer format number, then each
/// feature not yet declared.
pub fn adds(&self) -> Vec<String> {
let mut out = Vec::new();
if self.to.format > self.from.format {
out.push(format!("journal format {}", self.to.format));
}
out.extend(
self.to
.features
.iter()
.filter(|f| !self.from.has_feature(f))
.map(|f| format!("the journal feature {f}")),
);
out
}
}
impl JournalFormat {
/// Whether the books declared the feature `feature`.
pub fn has_feature(&self, feature: &str) -> bool {
self.features.iter().any(|f| f == feature)
}
/// Refuse a write that needs journal format `format` (or a later one)
/// in books that haven't declared it: they wait for their format
/// upgrade (FormatUpgradePending; [`crate::Store`]'s writes ask this
/// before they journal anything).
pub fn require_format(&self, format: u32) -> Result<(), StoreError> {
if self.format >= format {
Ok(())
} else {
Err(StoreError::FormatUpgradePending {
need: format!("journal format {format}"),
})
}
}
/// Refuse a write that needs the feature `feature` in books that
/// haven't declared it ([`Self::require_format`]'s kin).
pub fn require_feature(&self, feature: &str) -> Result<(), StoreError> {
if self.has_feature(feature) {
Ok(())
} else {
Err(StoreError::FormatUpgradePending {
need: format!("the journal feature {feature}"),
})
}
}
/// The format upgrade books in this format wait for, if any (rules.md
/// B-27, plan/served.md 3.9, D-39).
///
/// ```text
/// GOAL: books an older version wrote are never moved to a newer
/// format, which that version can't open, without the owner's
/// go-ahead, and whether they wait for one is a fact of their
/// journal alone.
/// SOURCE: plan/served.md 3.9 ("Format changes are the server's gate,
/// not the updater's": a server that would write a newer
/// journal format than a book has opens it on its old format
/// until the owner consents); D-39 (asked at every open until
/// given).
/// INVARIANTS:
/// 1. Some exactly when this version writes a higher format number,
/// or knows a feature the books haven't declared.
/// 2. Its `to` is this version's format (current()): every feature
/// it knows, so a book's format moves exactly once per version.
/// SCENARIOS:
/// 1. IF the books declared nothing (format 1, written before the
/// format gate): SHOULD wait for the upgrade to format 2.
/// 2. IF they declared this version's format and every feature it
/// knows: SHOULD wait for none.
/// 3. IF they declared this format without a feature this version
/// knows: SHOULD wait for one that adds it.
/// ```
pub fn upgrade(&self) -> Option<FormatUpgrade> {
self.upgrade_to(current())
}
/// [`Self::upgrade`] for a version whose format is `to`.
fn upgrade_to(&self, to: JournalFormat) -> Option<FormatUpgrade> {
let behind = self.format < to.format || to.features.iter().any(|f| !self.has_feature(f));
behind.then(|| FormatUpgrade {
from: self.clone(),
to,
})
}
}
/// Read a `journal.format_set` line, refusing a format or feature this
/// version doesn't read.
pub(crate) fn declared(line: &JournalLine) -> Result<JournalFormat, StoreError> {
@@ -570,6 +691,136 @@ mod tests {
);
}
/// This version's format: the newest it reads, every feature it
/// knows, each once, sorted (what new books and an upgrade declare).
#[test]
fn this_versions_format_is_the_newest_with_every_known_feature() {
let c = current();
assert_eq!(c.format, SUPPORTED_JOURNAL_FORMAT);
let mut known: Vec<String> = KNOWN_FEATURES.iter().map(|f| f.to_string()).collect();
known.sort();
known.dedup();
assert_eq!(c.features, known);
// What it declares reads back as itself.
let features: Vec<&str> = c.features.iter().map(String::as_str).collect();
let payload = JournalFormat::default()
.declaration(c.format, &features)
.unwrap();
assert_eq!(declared(&line(FORMAT_SET, payload)).unwrap(), c);
assert_eq!(c.upgrade(), None, "this version's books wait for nothing");
}
/// Books wait for an upgrade exactly when this version writes a higher
/// format number or knows a feature they haven't declared; the upgrade
/// is always to this version's whole format (SCENARIOS 1 to 3).
#[test]
fn books_wait_for_an_upgrade_exactly_when_behind() {
let at = |format: u32, features: &[&str]| JournalFormat {
format,
features: features.iter().map(|f| f.to_string()).collect(),
};
// Format 1 (nothing declared): waits for this version's format.
let old = JournalFormat::default();
let up = old.upgrade().expect("format 1 books wait for format 2");
assert_eq!((up.from.clone(), up.to.clone()), (old, current()));
assert_eq!(
up.adds()[0],
format!("journal format {SUPPORTED_JOURNAL_FORMAT}")
);
// A version that knows features: each one missing makes books wait,
// and the upgrade adds all of them at once.
let to = at(2, &["a", "b"]);
for (books, adds) in [
(
at(1, &[]),
vec![
"journal format 2",
"the journal feature a",
"the journal feature b",
],
),
(
at(2, &[]),
vec!["the journal feature a", "the journal feature b"],
),
(at(2, &["b"]), vec!["the journal feature a"]),
(at(2, &["a"]), vec!["the journal feature b"]),
] {
let up = books.upgrade_to(to.clone()).expect("behind");
assert_eq!(up.to, to, "the whole format, at once");
assert_eq!(up.adds(), adds, "{books:?}");
}
assert_eq!(at(2, &["a", "b"]).upgrade_to(to.clone()), None);
assert_eq!(
at(2, &["b", "a"]).upgrade_to(to.clone()),
None,
"in any order"
);
// Nothing ahead of this version counts as behind (it never opens).
assert_eq!(at(3, &["a", "b"]).upgrade_to(to), None);
}
/// A write that needs a format or feature the books haven't declared
/// is refused as waiting for the upgrade, saying what it needs; one
/// the books declared passes.
#[test]
fn a_write_needs_what_the_books_declared() {
let at = |format: u32, features: &[&str]| JournalFormat {
format,
features: features.iter().map(|f| f.to_string()).collect(),
};
let need = |r: Result<(), StoreError>| match r {
Err(StoreError::FormatUpgradePending { need }) => need,
other => panic!("expected FormatUpgradePending, got {other:?}"),
};
assert_eq!(need(at(1, &[]).require_format(2)), "journal format 2");
at(2, &[]).require_format(2).unwrap();
at(2, &[]).require_format(1).unwrap();
assert_eq!(
need(at(2, &["a"]).require_feature("b")),
"the journal feature b"
);
at(2, &["a", "b"]).require_feature("b").unwrap();
// The refusal names the guard, says nothing changed, and that only
// the owner can agree, and how.
let text = StoreError::FormatUpgradePending {
need: "journal format 2".into(),
}
.to_string();
for says in [
"format_upgrade_pending:",
"needs journal format 2",
"Nothing was changed",
"Only the owner can give it",
"owner passphrase",
"wanderbooks books upgrade",
"no tool can",
] {
assert!(text.contains(says), "{says}: {text}");
}
}
/// An upgrade never weakens the gate: a version that knows fewer
/// features than the books declared still stops at the declaration,
/// as newer, and one that knows them reads it.
#[test]
fn a_reader_that_doesnt_know_a_declared_feature_still_refuses_it_as_newer() {
let l = line(FORMAT_SET, json!({"format": 2, "features": ["a"]}));
match declared_knowing(&l, &[]) {
Err(StoreError::NewerJournal { need, .. }) => {
assert_eq!(need, "journal features a")
}
other => panic!("expected NewerJournal, got {other:?}"),
}
assert_eq!(
declared_knowing(&l, &["a"]).unwrap(),
JournalFormat {
format: 2,
features: vec!["a".into()]
}
);
}
/// A format newer than this version reads is newer whatever shape the
/// rest of its declaration takes, and however large its number: a
/// newer format may define its features differently, so they are
crates/ledger-store/src/lib.rs
+3 -3
View File
@@ -43,11 +43,11 @@ pub mod uploads;
pub use blob::BlobStore;
pub use error::StoreError;
pub use format::{JournalFormat, SUPPORTED_JOURNAL_FORMAT};
pub use format::{FormatUpgrade, JournalFormat, SUPPORTED_JOURNAL_FORMAT};
pub use journal::{GENESIS_PREV, JournalLine, JournalWriter};
pub use snapshot::{DiskCheck, Snapshot};
pub use store::{
Attachment, DB_FILE, DepreciationRun, DisposalPreview, EntryRow, EventRow, JOURNAL_FILE,
Attachment, DepreciationRun, DisposalPreview, EntryRow, EventRow, JOURNAL_FILE, LEGACY_DB_FILE,
LOCK_FILE, MAX_RECURRING_RUN, PayrollLineInput, Reconciliation, Statement, Store, Subject,
VerifyReport,
VerifyReport, database_in_use, db_file, db_file_for, is_database_file,
};
crates/ledger-store/src/root.rs
+18 -5
View File
@@ -665,7 +665,9 @@ impl Root {
/// built in a staging folder: every stored file re-hashed against its
/// name, the copy opened (its database rebuilt from the journal) with
/// the source's head, every file it names present, verified end to
/// end, given its settings if the source keeps none (and verified
/// end, moved to this version's journal format if the source's is
/// older (the copy is new books of this version; the source keeps its
/// own), given its settings if the source keeps none (and verified
/// again), synced, and only then renamed into place.
pub fn adopt(
&self,
@@ -781,15 +783,22 @@ impl Root {
}
store.verify_and_heal()?;
let blobs_verified = store.verify_blobs()?;
// The copy is new books of this version, in its journal format
// (the original, left as it was, keeps its own): it waits for
// no format upgrade, and its settings can move in.
let format_upgraded = store.upgrade_format()?.is_some();
let (settings_imported, settings_ignored) = match (&options.settings, keeps_settings) {
(Some((changes, source)), false) => {
let n = store.adopt_settings(changes, source)?.len();
store.verify_and_heal()?;
(n, false)
}
(Some(_), true) => (0, true),
(None, _) => (0, false),
};
// Verified again for what was written on the way in.
if store.head() != head {
store.verify_and_heal()?;
}
let final_head = store.head();
drop(store);
sync_dir(&staging)?;
@@ -806,6 +815,7 @@ impl Root {
settings_imported,
settings_ignored,
source_lock_created,
format_upgraded,
})
})();
let renamed = built.and_then(|report| {
@@ -1660,9 +1670,9 @@ pub struct AdoptReport {
pub id: BookId,
/// The source's head, `(seq, hash)`, verified before copying.
pub source_head: (u64, String),
/// The adopted books' head: the source's, plus (when settings were
/// moved in) their `settings.set`, after the journal format they need
/// if the source's is older.
/// The adopted books' head: the source's, plus this version's journal
/// format if the source's is older (`format_upgraded`), plus (when
/// settings were moved in) their `settings.set`.
pub head: (u64, String),
/// Stored files copied, each re-hashed against its name.
pub blob_files: u64,
@@ -1686,6 +1696,9 @@ pub struct AdoptReport {
pub settings_ignored: bool,
/// The source had no lock file, so taking its lock made one.
pub source_lock_created: bool,
/// The copy was moved to this version's journal format (the source's
/// was older; the source keeps its own).
pub format_upgraded: bool,
}
/// The source's writer lock: its lock file opened read-only (so a
crates/ledger-store/src/snapshot.rs
+4 -3
View File
@@ -19,7 +19,7 @@ use crate::error::StoreError;
use crate::format::JournalFormat;
use crate::journal::{GENESIS_PREV, JournalLine};
use crate::store::{
Attachment, BLOBS_DIR, DB_FILE, JOURNAL_FILE, Reconciliation, Statement, Store, Subject,
Attachment, BLOBS_DIR, JOURNAL_FILE, Reconciliation, Statement, Store, Subject,
apply_line_to_sqlite, compare_projections, projection_rows, query_attachment_links,
query_attachments, query_reconciliations, query_statements, read_attachment_bytes,
replay_line_into_ledger,
@@ -292,7 +292,8 @@ impl Snapshot {
/// by the concurrency test, 2026-10-13, as a false "differs").
pub fn check_disk(&self) -> DiskCheck {
use rusqlite::OpenFlags;
let path = self.data_dir.join(DB_FILE);
// The file this version's writer keeps (store::database_in_use).
let path = crate::store::database_in_use(&self.data_dir);
if !path.exists() {
return DiskCheck::Missing;
}
@@ -374,7 +375,7 @@ impl Snapshot {
/// `file:` URI for `path`, read-only and immutable (sqlite.org/uri.html):
/// every byte outside the unreserved set is percent-encoded.
fn immutable_uri(path: &Path) -> String {
pub(crate) fn immutable_uri(path: &Path) -> String {
let mut out = String::from("file:");
for b in path.to_string_lossy().bytes() {
if b.is_ascii_alphanumeric() || b"/-_.~".contains(&b) {
crates/ledger-store/src/store.rs
+291 -98
View File
@@ -52,8 +52,101 @@ use std::path::{Path, PathBuf};
/// File names inside the data directory.
pub const JOURNAL_FILE: &str = "journal.jsonl";
/// File names inside the data directory.
pub const DB_FILE: &str = "wanderbooks.db";
/// The projection database's one file before each schema had a file of
/// its own (every version through P2, whose schema was 12). This version
/// uses it only when it already holds this version's schema, so books that
/// need no migration aren't rebuilt for nothing; otherwise it is left as
/// it is, byte for byte, for the version that wrote it ([`db_file_for`]).
pub const LEGACY_DB_FILE: &str = "wanderbooks.db";
/// The projection database's file for schema `schema`:
/// `wanderbooks.v{schema}.db`.
///
/// ```text
/// GOAL: going back to an older version always finds the books it can
/// open (plan/served.md 3.9: before consent, the way back is always
/// there), however far this version's projection has moved.
/// SOURCE: plan/served.md 3.9, the lead's decision (2026-10-21: each
/// schema in a file of its own); rules.md B-27.
/// INVARIANTS:
/// 1. A version opens or creates only its own schema's file (or the
/// legacy file when that already holds its schema); a missing one is
/// built from the journal.
/// 2. It never migrates, writes or deletes a file of another schema,
/// older or newer: each is left byte for byte for its own version.
/// 3. The name depends only on the schema number, so versions agree on
/// it whatever number each was given.
/// ```
pub fn db_file_for(schema: u32) -> String {
format!("wanderbooks.v{schema}.db")
}
/// This version's own projection file: [`db_file_for`] its schema
/// ([`crate::schema::SCHEMA_VERSION`]).
pub fn db_file() -> String {
db_file_for(crate::schema::SCHEMA_VERSION)
}
/// Whether `name` is a projection database's file, of any schema (the
/// legacy one included): what [`Store::other_databases`] lists.
pub fn is_database_file(name: &str) -> bool {
name == LEGACY_DB_FILE
|| name
.strip_prefix("wanderbooks.v")
.and_then(|rest| rest.strip_suffix(".db"))
.is_some_and(|n| !n.is_empty() && n.bytes().all(|b| b.is_ascii_digit()))
}
/// The projection database this version uses for the books in `data_dir`
/// ([`db_file_for`] INVARIANT 1): its own schema's file if there is one;
/// else the legacy file if that already holds this version's schema; else
/// its own schema's file, to be built from the journal. Read only: a file
/// of another schema is looked at, never opened for writing.
pub fn database_in_use(data_dir: &Path) -> PathBuf {
let own = data_dir.join(db_file());
if std::fs::symlink_metadata(&own).is_ok() {
return own;
}
let legacy = data_dir.join(LEGACY_DB_FILE);
if schema_of(&legacy) == Some(crate::schema::SCHEMA_VERSION) {
legacy
} else {
own
}
}
/// The schema a projection file says it holds, read without writing a
/// byte of it (an immutable read when no writer has it open; None when it
/// isn't there, isn't a database, or says none).
pub(crate) fn schema_of(path: &Path) -> Option<u32> {
use rusqlite::OpenFlags;
if !path.is_file() {
return None;
}
let conn = if path.with_extension("db-wal").exists() {
Connection::open_with_flags(
path,
OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
)
} else {
Connection::open_with_flags(
crate::snapshot::immutable_uri(path),
OpenFlags::SQLITE_OPEN_READ_ONLY
| OpenFlags::SQLITE_OPEN_NO_MUTEX
| OpenFlags::SQLITE_OPEN_URI,
)
}
.ok()?;
conn.query_row(
"SELECT value FROM meta WHERE key='schema_version'",
[],
|r| r.get::<_, String>(0),
)
.ok()?
.parse()
.ok()
}
/// The lock file a writer holds while the books are open. Readers (the
/// viewer's [`crate::Snapshot`]) never take it.
@@ -250,10 +343,6 @@ pub struct Store {
base_currency: Currency,
/// The journal format in force (the format gate's record).
format: JournalFormat,
/// Refuse every journal feature not yet in force
/// ([`Self::refuse_new_features`]): a stand-in for a format-upgrade
/// gate that refuses until the owner consents.
refuse_features: bool,
}
/// One filing of bytes into the blob store and journal.
@@ -275,24 +364,44 @@ impl Store {
/// Create a new store. Fails if one already exists.
///
/// ```text
/// GOAL: start empty books in a data directory.
/// SOURCE: docs/receipt-store.md "Line format" (genesis prev); R-11.
/// GOAL: start empty books in a data directory, in this version's
/// journal format.
/// SOURCE: docs/receipt-store.md "Line format" (genesis prev); R-11;
/// plan/served.md 3.9 and D-39 (books this version makes wait
/// for no format upgrade).
/// INVARIANTS:
/// 1. The journal's first line MUST be `store.created` with the base
/// currency, chained from the genesis hash (64 zeros).
/// 2. The writer lock MUST be held from creation on.
/// 3. The second line declares this version's journal format, every
/// feature it knows (format::current()): the new books wait for
/// no format upgrade.
/// SCENARIOS:
/// 1. IF a journal already exists there: SHOULD refuse
/// - NEVER overwrite or append to existing books.
/// ```
pub fn create(data_dir: &Path, base_currency: Currency) -> Result<Self, StoreError> {
Self::create_in_format(data_dir, base_currency, &crate::format::current())
}
/// [`Self::create`], declaring the journal format `format` (format 1,
/// [`JournalFormat::default`], declares nothing): books as an older
/// version made them, for tests of their format upgrade. Everything
/// else creates books with [`Self::create`].
#[doc(hidden)]
pub fn create_in_format(
data_dir: &Path,
base_currency: Currency,
format: &JournalFormat,
) -> Result<Self, StoreError> {
let journal_path = data_dir.join(JOURNAL_FILE);
if journal_path.exists() {
return Err(StoreError::AlreadyExists(data_dir.display().to_string()));
}
std::fs::create_dir_all(data_dir)?;
let lock = lock_books(data_dir)?;
let db_path = data_dir.join(DB_FILE);
// New books: this version's own schema's file (db_file_for).
let db_path = data_dir.join(db_file());
let conn = Self::open_conn(&db_path)?;
Self::migrate(&conn)?;
Self::write_meta(&conn, "base_currency", base_currency.code())?;
@@ -307,13 +416,16 @@ impl Store {
ledger: Ledger::new(base_currency),
base_currency,
format: JournalFormat::default(),
refuse_features: false,
};
let line = store.journal.append(
"store.created",
json!({ "base_currency": base_currency.code() }),
)?;
apply_line_to_sqlite(&store.conn, &line)?;
if *format != JournalFormat::default() {
let features: Vec<&str> = format.features.iter().map(String::as_str).collect();
store.declare_journal_format(format.format, &features)?;
}
Ok(store)
}
@@ -352,6 +464,13 @@ impl Store {
/// for forensics and rebuild from the journal
/// BECAUSE the database is derived; the journal is the record.
/// 7. IF the database is behind the journal: SHOULD apply the rest.
/// 8. The database is this version's own schema's file, or the
/// legacy one when that already holds this version's schema
/// ([`database_in_use`]); a file of any other schema, older or
/// newer, is left byte for byte, never migrated, refused or
/// deleted; this version's own missing: SHOULD build it from the
/// journal (SCENARIO 5). One named as this version's that holds
/// another schema is damaged (SCENARIO 6).
/// ```
pub fn open(data_dir: &Path) -> Result<Self, StoreError> {
let journal_path = data_dir.join(JOURNAL_FILE);
@@ -385,12 +504,17 @@ impl Store {
None
};
let journal = JournalWriter::from_lines(&journal_path, &lines)?;
let db_path = data_dir.join(DB_FILE);
// This version's own schema's file (SCENARIO 8): another schema's
// is never opened for writing.
let db_path = database_in_use(data_dir);
let mut fresh_db = !db_path.exists();
let schema = crate::schema::SCHEMA_VERSION;
let damage = if fresh_db {
None
} else if !Self::db_file_healthy(&db_path) {
Some("failed integrity_check".to_string())
} else if let Some(other) = schema_of(&db_path).filter(|v| *v != schema) {
Some(format!("holds schema {other}, not this version's {schema}"))
} else {
match Self::db_currency(&db_path) {
Some(c) if c == base_currency => None,
@@ -433,7 +557,6 @@ impl Store {
ledger,
base_currency,
format,
refuse_features: false,
};
store.catch_up(&lines)?;
if let Some(t) = torn {
@@ -579,9 +702,35 @@ impl Store {
&self.format
}
/// The projection database's file this version uses, by name
/// ([`database_in_use`]): what `books.verify` and `wanderbooks doctor`
/// say.
pub fn database_file(&self) -> String {
file_name_of(&self.db_path)
}
/// The other projection files beside it (other schemas', the legacy
/// one if unused), by name, sorted: listed, never deleted
/// ([`db_file_for`] INVARIANT 2).
pub fn other_databases(&self) -> Vec<String> {
let in_use = self.database_file();
let mut out: Vec<String> = std::fs::read_dir(&self.data_dir)
.into_iter()
.flatten()
.flatten()
.filter(|e| e.file_type().is_ok_and(|t| t.is_file()))
.map(|e| e.file_name().to_string_lossy().into_owned())
.filter(|n| is_database_file(n) && *n != in_use)
.collect();
out.sort();
out
}
/// Declare the journal format: journal a `journal.format_set` line.
/// A writer declares the format before the first event that needs it,
/// so an older reader stops at the declaration and says it is too old.
/// so an older reader stops at the declaration and says it is too old:
/// new books as they are created ([`Self::create`]), older books at
/// their format upgrade ([`Self::upgrade_format`]), never a tool.
///
/// ```text
/// GOAL: books that tell every reader which format they are in.
@@ -613,74 +762,92 @@ impl Store {
// items, approvals (journal format 2, crate::meta)
// ------------------------------------------------------------------
/// Declare journal format 2 before the first event that needs it, so
/// an older reader stops at the declaration (plan/served.md 5.1).
fn ensure_format(&mut self) -> Result<(), StoreError> {
if self.format.format < crate::meta::FORMAT {
let features = self.format.features.clone();
let features: Vec<&str> = features.iter().map(String::as_str).collect();
self.declare_journal_format(crate::meta::FORMAT, &features)?;
}
Ok(())
/// Refuse an event that needs journal format 2 in books that haven't
/// declared it: books an older version wrote, waiting for their format
/// upgrade (the owner's go-ahead, [`Self::upgrade_format`]). Books this
/// version made declared it when they were created.
fn ensure_format(&self) -> Result<(), StoreError> {
self.format.require_format(crate::meta::FORMAT)
}
/// Declare a journal feature (crate::format::KNOWN_FEATURES) before the
/// first line that needs it: a new event type, or a field or value on
/// a known event type that an older version would otherwise misread
/// silently. Declared with the format in force (at least format 2,
/// crate::meta::FORMAT) and every feature already in force, so an
/// older reader stops at the declaration (rules.md B-27); nothing is
/// journaled when it is already declared. Books that never use the
/// feature stay readable by older versions.
/// Refuse a write that needs the journal feature `feature`
/// (crate::format::KNOWN_FEATURES: a field or value on an event type an
/// older version knows, which that version would misread silently)
/// unless the books declared it. Books this version made declared it
/// when they were created; older books declare it only at their format
/// upgrade, with every other feature this version knows, never one at
/// a time and never without the owner's go-ahead
/// ([`Self::upgrade_format`]). Ask it only for a write that uses the
/// feature (the line about to be journaled holds the new field or
/// value): a write that doesn't is never refused, since declining the
/// upgrade keeps everything else working (D-39).
///
/// Each command here that writes a feature's line declares it first.
/// A caller that journals several commands as one (a tool drafting an
/// Each command here that writes a feature's line asks for it first. A
/// caller that journals several commands as one (a tool drafting an
/// entry and tagging it) calls this before the first of them, once
/// every check has passed: a declaration never lands after another
/// line of the same command (where a refusal to upgrade the format
/// would leave the command half done), and a refused command leaves
/// none.
/// every check has passed: a refusal never lands after another line of
/// the same command (which would leave the command half done), and a
/// refused command leaves none.
///
/// ```text
/// GOAL: an older wanderbooks stops at the declaration (NewerJournal)
/// instead of misreading, or failing on, a line it can't read.
/// SOURCE: rules.md B-27; plan/served.md 5.1 (format gate).
/// instead of misreading a line it thinks it knows, and books
/// move past what their owner's older version reads only when
/// the owner says so.
/// SOURCE: rules.md B-27; plan/served.md 5.1 (format gate), 3.9 and
/// D-39 (the format upgrade).
/// SCENARIOS:
/// 1. IF the books declared it: SHOULD pass.
/// 2. IF not: SHOULD refuse (FormatUpgradePending), journaling
/// nothing.
/// ```
pub fn ensure_feature(&mut self, feature: &str) -> Result<(), StoreError> {
self.ensure_features(&[feature])
pub fn ensure_feature(&self, feature: &str) -> Result<(), StoreError> {
debug_assert!(
crate::format::KNOWN_FEATURES.contains(&feature),
"{feature} isn't a feature this version knows"
);
self.format.require_feature(feature)
}
/// [`Self::ensure_feature`] for several at once: one declaration
/// naming every one not yet in force, or none when all are.
fn ensure_features(&mut self, needed: &[&str]) -> Result<(), StoreError> {
let mut features: Vec<String> = self.format.features.clone();
let before = features.len();
for f in needed {
if !features.iter().any(|g| g == f) {
features.push(f.to_string());
}
}
if features.len() == before {
return Ok(());
}
if self.refuse_features {
return Err(StoreError::FormatRefused(format!(
"journal features {:?}: these books take no new feature now",
&features[before..]
)));
}
let format = self.format.format.max(crate::meta::FORMAT);
let features: Vec<&str> = features.iter().map(String::as_str).collect();
self.declare_journal_format(format, &features)
/// The format upgrade (plan/served.md 3.9, D-39): declare this
/// version's journal format, with every feature it knows, in one
/// `journal.format_set`, for books an older version wrote. What was
/// upgraded; None (journaling nothing) when the books already are in
/// this version's format. The books' figures don't change: only what
/// may be written from here on.
///
/// Only the owner's go-ahead comes here: the books server, after the
/// owner passphrase and a verified backup with the keys (ledger-mcp
/// local.rs and backup.rs); and an adoption, whose copy is new books
/// of this version while the original keeps its own format (root.rs).
/// No tool does.
///
/// ```text
/// GOAL: older books move to this version's format at once, on the
/// record, exactly once.
/// SOURCE: plan/served.md 3.9 ("Format changes are the server's
/// gate"); D-39; rules.md B-27.
/// INVARIANTS:
/// 1. One journal line declares the whole format: the format moves
/// exactly once, never feature by feature.
/// 2. Books already in this version's format: nothing is journaled.
/// 3. The ledger is unchanged: every report reads as before.
/// ```
pub fn upgrade_format(&mut self) -> Result<Option<crate::format::FormatUpgrade>, StoreError> {
let Some(upgrade) = self.format.upgrade() else {
return Ok(None);
};
let features: Vec<&str> = upgrade.to.features.iter().map(String::as_str).collect();
self.declare_journal_format(upgrade.to.format, &features)?;
debug_assert_eq!(self.format, upgrade.to);
Ok(Some(upgrade))
}
/// Refuse (`refuse: true`) or take again every journal feature not yet
/// in force, as a format-upgrade gate does that waits for the owner's
/// consent: what a command does when [`Self::ensure_feature`] refuses
/// (it journals nothing). Not saved: the books open taking features.
#[doc(hidden)]
pub fn refuse_new_features(&mut self, refuse: bool) {
self.refuse_features = refuse;
/// [`Self::ensure_feature`] for each feature a write needs: refused
/// (FormatUpgradePending, journaling nothing) unless the books declared
/// every one.
fn ensure_features(&self, needed: &[&str]) -> Result<(), StoreError> {
needed.iter().try_for_each(|f| self.ensure_feature(f))
}
/// Change settings (`settings.set`): all or nothing, each value
@@ -719,9 +886,10 @@ impl Store {
if changes.is_empty() && !(adopt && !self.ledger.settings().is_journaled()) {
return Ok(changes);
}
// A key an older version doesn't know needs its feature declared
// first (crate::format::setting_feature), in the one declaration
// that also brings the books to format 2.
// A key an older version doesn't know needs its feature
// (crate::format::setting_feature), and every settings.set format
// 2: refused, journaling nothing, in books that haven't declared
// them (their format upgrade does).
let needed: Vec<&str> = changes
.iter()
.filter_map(|c| crate::format::setting_feature(&c.key))
@@ -1451,10 +1619,12 @@ impl Store {
/// Name a party the creditor of a liability account the company owes
/// it itself on, or stop (`on` false): ledger_core's
/// `Ledger::set_party_creditor`. Journaled as `party.creditor_set
/// {id, account_id, on}`, after the journal declares the feature it
/// needs ([`crate::format::PARTY_CREDITORS`]) the first time, so an
/// older wanderbooks stops there and says it is too old instead of
/// misreading the books.
/// {id, account_id, on}`, which needs the journal feature
/// [`crate::format::PARTY_CREDITORS`], so an older wanderbooks stops at
/// the declaration and says it is too old instead of misreading the
/// books. Books that haven't declared it (an older version's, until
/// their format upgrade) refuse it, FormatUpgradePending, journaling
/// nothing.
pub fn set_party_creditor(
&mut self,
id: ledger_core::PartyId,
@@ -1820,8 +1990,10 @@ impl Store {
/// 3. Nothing due (the month rounds to nothing, a later run caught it
/// up, the life is over and posted) journals nothing: an entry
/// can't carry a zero line (P3 depreciation review R10).
/// 4. A period run before and reversed is run again only after
/// journal feature month_reposting is declared.
/// 4. A period run before and reversed is run again only in books
/// that declare journal feature month_reposting (books an older
/// version wrote: after their format upgrade); refused before
/// anything is journaled otherwise.
/// SCENARIOS:
/// 1. IF the asset is unknown or disposed, the period malformed, or
/// it ends before the asset was placed in service: SHOULD refuse.
@@ -2344,8 +2516,8 @@ impl Store {
)?;
// The disposal month's run was reversed (else the final
// depreciation would be in the disposal entry): its line again,
// which needs a journal feature, declared (or refused) before the
// first line is journaled.
// which needs a journal feature, asked for (and refused in books
// that haven't declared it) before the first line is journaled.
if depreciation.is_some() && self.depreciation_period_was_run(asset_id, &preview.period)? {
self.ensure_feature(crate::format::MONTH_REPOSTING)?;
}
@@ -2447,17 +2619,19 @@ impl Store {
// Rehearsed whole: a doomed close journals nothing, not even the
// Retained Earnings account creation below.
self.rehearse(|l| l.close_period_to(id, to))?;
if matches!(to, CloseTo::Members(_) | CloseTo::MembersCumulative { .. }) {
// A close counted from its regime's start: an older version
// would replay it under the run rule (format gate, B-27).
// Refused before anything is journaled until the books
// declare it.
self.ensure_feature(crate::format::MEMBERS_CUMULATIVE)?;
}
if matches!(to, CloseTo::RetainedEarnings { .. }) {
// Retained Earnings must exist before the close so its creation
// (if needed) is journaled as account.added; the close's own
// ensure then no-ops and replay stays deterministic.
self.ensure_control_account(&ledger_core::RETAINED_EARNINGS)?;
}
if matches!(to, CloseTo::Members(_) | CloseTo::MembersCumulative { .. }) {
// A close counted from its regime's start: an older version
// would replay it under the run rule (format gate, B-27).
self.ensure_feature(crate::format::MEMBERS_CUMULATIVE)?;
}
let mut next = self.ledger.clone();
let closing_entry = next.close_period_to(id, to)?;
let entry_detail = match closing_entry {
@@ -3315,10 +3489,11 @@ impl Store {
}
/// Register an amortization schedule. Journaled as `schedule.registered`,
/// with its `convention` (`daily` or `monthly_even`). The first daily
/// schedule declares journal feature `schedule_convention` before its
/// line: an older version reads every schedule as equal months, and
/// would refuse a daily one's postings as wrong (rather than as newer).
/// with its `convention` (`daily` or `monthly_even`). A daily schedule
/// needs journal feature `schedule_convention` (refused, journaling
/// nothing, in books that haven't declared it): an older version reads
/// every schedule as equal months, and would refuse a daily one's
/// postings as wrong (rather than as newer).
pub fn register_schedule(
&mut self,
new: ledger_core::NewSchedule,
@@ -3352,8 +3527,9 @@ impl Store {
/// Post one month of a schedule. Journaled as `schedule.posted` with the
/// whole entry, so a month is never half-posted or posted twice; replay
/// re-derives the entry and checks it line by line. A month a reversal
/// un-posted posts again only after journal feature `month_reposting` is
/// declared: an older version would refuse it as posted twice.
/// un-posted posts again only in books that declare journal feature
/// `month_reposting` (refused, journaling nothing, otherwise): an older
/// version would refuse it as posted twice.
pub fn post_schedule_month(
&mut self,
id: ledger_core::ScheduleId,
@@ -3535,10 +3711,9 @@ impl Store {
/// Give an account a new name (see `Ledger::rename_account`), with an
/// optional reason. Journaled as `account.renamed`, keyed by the
/// account's id (a code can change; an id can't), with its code, the
/// new name, the name it had and the reason. The first rename declares
/// the journal feature `account_renamed`
/// ([`crate::format::FEATURE_ACCOUNT_RENAMED`]), so an earlier version
/// stops at the declaration.
/// new name, the name it had and the reason. It needs the journal
/// feature `account_renamed` ([`crate::format::FEATURE_ACCOUNT_RENAMED`]),
/// so an earlier version stops at the declaration.
///
/// ```text
/// GOAL: rename an account, keeping the name it had in the journal.
@@ -3549,8 +3724,9 @@ impl Store {
/// 1. Only the account's name changes (Ledger::rename_account
/// INVARIANT 1): every entry, line, hash and earlier journal line
/// stays as it was.
/// 2. Rehearsed and checked first: a refusal journals nothing, not
/// even the feature's declaration.
/// 2. Rehearsed and checked first: a refusal journals nothing (and
/// books that haven't declared `account_renamed` refuse it,
/// FormatUpgradePending, until their format upgrade).
/// ```
pub fn rename_account(
&mut self,
@@ -3659,7 +3835,9 @@ impl Store {
/// 2. IF the core refuses it: SHOULD refuse, journaling nothing.
/// 3. IF it changes no decision: SHOULD journal nothing but a
/// changed W-9 flag (Store::set_vendor_w9_flag).
/// 4. Otherwise: SHOULD declare the feature once, then journal it.
/// 4. Otherwise: SHOULD journal it, in books that declare the feature
/// (refused, FormatUpgradePending, journaling nothing, in books
/// that don't).
/// ```
pub fn set_vendor_1099(
&mut self,
@@ -4638,6 +4816,8 @@ impl Store {
.map(|l| sha256_hex(l.to_canonical().as_bytes()))
.unwrap_or_else(|| GENESIS_PREV.to_string()),
healed,
database: self.database_file(),
other_databases: self.other_databases(),
})
}
@@ -6960,6 +7140,19 @@ pub struct VerifyReport {
pub head: String,
/// Why the projection was rebuilt, if it was.
pub healed: Option<String>,
/// The projection database's file verified: this version's
/// ([`Store::database_file`]).
pub database: String,
/// Other schemas' projection files beside it, left as they are
/// ([`Store::other_databases`]).
pub other_databases: Vec<String>,
}
/// A path's last part, as text.
fn file_name_of(path: &Path) -> String {
path.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_default()
}
/// A projection of `lines` into a private in-memory database, built with