diff --git a/crates/ledger-cli/tests/backups.rs b/crates/ledger-cli/tests/backups.rs new file mode 100644 index 0000000..ac343de --- /dev/null +++ b/crates/ledger-cli/tests/backups.rs @@ -0,0 +1,1388 @@ +//! The server's backups (plan 3.7, D-30, D-31): every book served from the +//! data root backs itself up to the targets its settings name, a while +//! after it is written, when it opens, before it closes, and when asked. +//! Oracles: the INVARIANTS of ledger-mcp backup.rs and ledger-store +//! backup.rs, and plan 3.7's named tests. +#![cfg(unix)] + +use ledger_mcp::admin::{self, Request, Response}; +use ledger_store::backup::{self, Git}; +use ledger_store::root::{BookId, Root}; +use rmcp::ServiceExt; +use rmcp::model::{CallToolRequestParams, ClientConfig}; +use rmcp::service::{RoleClient, RunningService}; +use rmcp::transport::StreamableHttpClientTransport; +use rmcp::transport::streamable_http_client::StreamableHttpClientTransportConfig; +use serde_json::{Value, json}; +use std::io::{BufRead, BufReader}; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant}; + +type Client = RunningService; + +/// A home with a data root made by `wanderbooks setup`, and a place for +/// the backups beside it. +struct Home { + dir: tempfile::TempDir, +} + +impl Home { + fn new(tag: &str) -> Self { + let dir = tempfile::Builder::new() + .prefix(&format!("wb-bk-{tag}-")) + .tempdir() + .unwrap(); + let out = Command::new(env!("CARGO_BIN_EXE_wanderbooks")) + .args(["setup", "--home"]) + .arg(dir.path()) + .env_remove("XDG_DATA_HOME") + .output() + .unwrap(); + assert!(out.status.success(), "{out:?}"); + Self { dir } + } + + fn root(&self) -> PathBuf { + self.dir.path().join(".local/share/wanderbooks") + } + + fn book_dir(&self, book: &str) -> PathBuf { + Root::open(&self.root()) + .unwrap() + .book_dir(&BookId::parse(book).unwrap()) + } + + fn admin(&self, request: Request) -> Response { + admin::call(&self.root().join("run/admin.sock"), request).unwrap() + } + + fn create(&self, id: &str) { + match self.admin(Request::Create { + id: id.into(), + currency: "USD".into(), + }) { + Response::Created { problem: None, .. } => {} + other => panic!("{other:?}"), + } + } + + fn token(&self, book: &str, scopes: &[&str]) -> String { + match self.admin(Request::TokenCreate { + book: book.into(), + name: format!("test-{}", scopes.join("-")), + scopes: scopes.iter().map(|s| s.to_string()).collect(), + replace: true, + }) { + Response::TokenCreated { text, .. } => text.0, + other => panic!("{other:?}"), + } + } + + /// A place outside the data root (a folder on "another disk"). + fn outside(&self, name: &str) -> PathBuf { + let p = self.dir.path().join("elsewhere").join(name); + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + p + } + + /// A bare repository standing for the private git remote. + fn remote(&self) -> PathBuf { + let p = self.outside("remote.git"); + let out = Command::new("git") + .args(["init", "-q", "--bare", "-b", "main"]) + .arg(&p) + .output() + .unwrap(); + assert!(out.status.success(), "{out:?}"); + p + } +} + +/// A running `wanderbooks server`, ended (kill) when dropped. +struct Server { + child: Child, +} + +impl Server { + /// Start one, backing up after `quiet` of no writing (a drill first + /// looked for after the server's own wait: a minute, past any test). + fn start(home: &Home, quiet: Duration) -> Self { + Self::start_with(home, quiet, None) + } + + /// [`Self::start`], a drill first looked for `drill_first` after books + /// are seen open. + fn start_with(home: &Home, quiet: Duration, drill_first: Option) -> Self { + let mut command = Command::new(env!("CARGO_BIN_EXE_wanderbooks")); + command + .arg("server") + .arg("--root") + .arg(home.root()) + .arg("--backup-quiet") + .arg(quiet.as_millis().to_string()); + if let Some(d) = drill_first { + command.arg("--drill-first").arg(d.as_millis().to_string()); + } + let mut child = command + .env_clear() + .env("PATH", "/usr/bin:/bin") + .env("HOME", home.dir.path()) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let mut line = String::new(); + BufReader::new(child.stdout.take().unwrap()) + .read_line(&mut line) + .unwrap(); + assert_eq!(line, "ready\n", "the server didn't say ready"); + Self { child } + } + + /// Close it as the host does (stdin ends) and wait for it to exit. + fn close(mut self) -> std::process::ExitStatus { + drop(self.child.stdin.take()); + let deadline = Instant::now() + Duration::from_secs(60); + loop { + if let Some(status) = self.child.try_wait().unwrap() { + return status; + } + assert!(Instant::now() < deadline, "the server didn't close"); + std::thread::sleep(Duration::from_millis(20)); + } + } +} + +impl Drop for Server { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +async fn connect(home: &Home, book: &str, token: String) -> Client { + // The short path is good in this process only, while it lives: kept. + let short = ledger_store::root::SocketPath::new(&home.root().join("run"), "mcp.sock").unwrap(); + let path = short.path().to_path_buf(); + std::mem::forget(short); + let http = reqwest::Client::builder() + .unix_socket(path) + .build() + .unwrap(); + let config = + StreamableHttpClientTransportConfig::with_uri(format!("http://localhost/b/{book}/mcp")) + .auth_header(token); + ClientConfig::default() + .serve(StreamableHttpClientTransport::with_client(http, config)) + .await + .unwrap() +} + +/// Call a tool: Ok(structured result) or Err(the error text). +async fn call(c: &Client, tool: &str, args: Value) -> Result { + let args = args.as_object().cloned().unwrap_or_default(); + let r = tokio::time::timeout( + Duration::from_secs(60), + c.call_tool(CallToolRequestParams::new(tool.to_string()).with_arguments(args)), + ) + .await + .map_err(|_| "no answer".to_string())? + .map_err(|e| format!("protocol: {e}"))?; + let text = r + .content + .iter() + .find_map(|c| c.as_text().map(|t| t.text.clone())) + .unwrap_or_default(); + if r.is_error == Some(true) { + Err(text) + } else { + Ok(r.structured_content + .unwrap_or_else(|| serde_json::from_str(&text).unwrap_or(Value::String(text)))) + } +} + +async fn head(c: &Client) -> u64 { + call(c, "books.start", json!({})).await.unwrap()["head"]["seq"] + .as_u64() + .unwrap() +} + +/// Name the backup targets in the book's settings. +async fn set_targets(c: &Client, remote: Option<&Path>, folder: Option<&Path>) { + let mut changes = Vec::new(); + if let Some(r) = remote { + changes.push(json!({"key": "backup.git_remote", "value": r.display().to_string()})); + } + if let Some(f) = folder { + changes.push(json!({"key": "backup.folder", "value": f.display().to_string()})); + } + call( + c, + "books.set_setting", + json!({"changes": changes, "decided_by": "Test Owner", "note": "Backups."}), + ) + .await + .unwrap(); +} + +/// Draft, receipt and post one $n entry. +async fn post_one(c: &Client, n: u64) { + for (code, name, ty) in [("1000", "Cash", "asset"), ("6000", "Supplies", "expense")] { + let _ = call( + c, + "books.add_account", + json!({"code": code, "name": name, "account_type": ty}), + ) + .await; + } + let e = call( + c, + "books.draft_entry", + json!({"date": "2026-03-01", "description": format!("entry {n}")}), + ) + .await + .unwrap(); + let id = e["entry_id"].as_u64().unwrap(); + for (account, side) in [("6000", "debit"), ("1000", "credit")] { + call( + c, + "books.add_line", + json!({"entry_id": id, "account": account, "side": side, "amount": format!("{n}.00")}), + ) + .await + .unwrap(); + } + call( + c, + "books.attach_receipt", + json!({"entry_id": id, "file_name": format!("r{n}.txt"), + "mime_type": "text/plain", "content_base64": "cmVjZWlwdA==", + "sha256": "6f32860910ca0fb2a20c7fda143666b09dbf8db5238195c90a586fb542ff0cad"}), + ) + .await + .unwrap(); + call( + c, + "books.post_entry", + json!({"entry_id": id, "confirmed": true, "no_party_reason": "test"}), + ) + .await + .unwrap(); +} + +/// Long enough for the server's backup loop (which looks every 2 s) to +/// have seen books just made, before they name any target: what it does +/// on first seeing them (a backup, a drill) is then done. +async fn settle() { + tokio::time::sleep(Duration::from_millis(4500)).await; +} + +/// Wait until every target holds the books' head (or `within` passes). +async fn backed_up(c: &Client, within: Duration) -> Value { + let deadline = Instant::now() + within; + loop { + let s = call(c, "books.backup", json!({})).await.unwrap(); + let head = s["head_seq"].as_u64().unwrap(); + let targets = s["targets"].as_array().unwrap(); + if !targets.is_empty() + && targets + .iter() + .all(|t| t["through_seq"].as_u64() == Some(head)) + { + return s; + } + assert!(Instant::now() < deadline, "not backed up in time: {s}"); + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + +/// Prove the backup at `source` whole; its head. +fn proven(home: &Home, source: &Path, tag: &str) -> (u64, String) { + let scratch = home.outside(&format!("scratch-{tag}")); + let v = backup::verify( + &Git::find().unwrap(), + &source.display().to_string(), + &scratch, + ) + .unwrap_or_else(|e| panic!("{tag}: {e}")); + v.head +} + +fn live_head(home: &Home, book: &str) -> (u64, String) { + let s = ledger_store::Snapshot::load(&home.book_dir(book)).unwrap(); + (s.journal_seq(), s.head_hash().to_string()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_book_backs_itself_up_to_both_targets_soon_after_a_write() { + let home = Home::new("both"); + let server = Server::start(&home, Duration::from_millis(300)); + home.create("acme"); + let c = connect( + &home, + "acme", + home.token("acme", &["read", "write", "post", "admin"]), + ) + .await; + // None set up: said, and nothing is stale. + let s = call(&c, "books.backup", json!({})).await.unwrap(); + assert_eq!(s["targets"], json!([])); + assert!( + s["note"].as_str().unwrap().contains("backup.git_remote"), + "{s}" + ); + // books.start says so too. + let steps = |v: Value| v["next_steps"].to_string(); + let start = steps(call(&c, "books.start", json!({})).await.unwrap()); + assert!(start.contains("No backups are set up"), "{start}"); + let (remote, folder) = (home.remote(), home.outside("disk")); + std::fs::create_dir_all(&folder).unwrap(); + set_targets(&c, Some(&remote), Some(&folder)).await; + post_one(&c, 7).await; + let s = backed_up(&c, Duration::from_secs(30)).await; + assert_eq!(s["note"], Value::Null); + let targets = s["targets"].as_array().unwrap(); + assert_eq!(targets.len(), 2, "{s}"); + for t in targets { + assert_eq!(t["stale"], false, "{t}"); + assert_eq!(t["last_error"], Value::Null, "{t}"); + assert!(t["last_ok_at"].is_string(), "{t}"); + } + let start = steps(call(&c, "books.start", json!({})).await.unwrap()); + assert!(!start.contains("books.backup"), "{start}"); + // backup_does_not_advance_head: the books' head is what was written. + let live = live_head(&home, "acme"); + assert_eq!(s["head_seq"].as_u64(), Some(live.0)); + // Each target holds the books whole, at their head. + assert_eq!(proven(&home, &remote, "remote"), live); + assert_eq!(proven(&home, &folder.join("acme.git"), "folder"), live); + // Its journal is the live journal's beginning (a drill's check). + let tree = home.outside("scratch-folder"); + assert!( + backup::is_prefix( + &home.book_dir("acme").join(ledger_store::JOURNAL_FILE), + &tree.join(ledger_store::JOURNAL_FILE) + ) + .unwrap() + ); + // Nothing an agent is shown names the data root. + assert!( + !s.to_string().contains(&home.root().display().to_string()), + "{s}" + ); + drop(c); + assert!(server.close().success()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn closing_backs_the_books_up_and_opening_catches_up() { + let home = Home::new("close"); + let folder = home.outside("disk"); + std::fs::create_dir_all(&folder).unwrap(); + // Too long a quiet to back up while running: only the close does. + let server = Server::start(&home, Duration::from_secs(3600)); + home.create("acme"); + settle().await; + let c = connect( + &home, + "acme", + home.token("acme", &["read", "write", "post", "admin"]), + ) + .await; + set_targets(&c, None, Some(&folder)).await; + post_one(&c, 3).await; + drop(c); + assert!(server.close().success()); + let live = live_head(&home, "acme"); + assert_eq!(proven(&home, &folder.join("acme.git"), "closed"), live); + + // archive_catches_up_at_start: a backup target named while the server + // was down (here: the folder was missing then made) is caught up when + // the books open. + let late = home.outside("late"); + let server = Server::start(&home, Duration::from_secs(3600)); + let c = connect( + &home, + "acme", + home.token("acme", &["read", "write", "post", "admin"]), + ) + .await; + set_targets(&c, None, Some(&late)).await; + // The folder doesn't exist: said plainly, the books unharmed. + let s = call(&c, "books.back_up_now", json!({})).await.unwrap(); + let err = s["targets"][0]["last_error"] + .as_str() + .unwrap_or_default() + .to_string(); + assert!(err.contains("folder"), "{s}"); + let start = call(&c, "books.start", json!({})).await.unwrap()["next_steps"].to_string(); + assert!(start.contains("behind or failing"), "{start}"); + assert!(!err.contains(&home.root().display().to_string()), "{err}"); + drop(c); + assert!(server.close().success()); + // The disk is connected while the books are closed: the books catch + // up when they open. + std::fs::create_dir_all(&late).unwrap(); + let server = Server::start(&home, Duration::from_secs(3600)); + let c = connect( + &home, + "acme", + home.token("acme", &["read", "write", "post", "admin"]), + ) + .await; + let s = backed_up(&c, Duration::from_secs(30)).await; + assert_eq!(s["targets"][0]["last_error"], Value::Null, "{s}"); + assert_eq!( + proven(&home, &late.join("acme.git"), "late"), + live_head(&home, "acme") + ); + drop(c); + assert!(server.close().success()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn backing_up_now_needs_admin_and_never_moves_the_head() { + let home = Home::new("now"); + let folder = home.outside("disk"); + std::fs::create_dir_all(&folder).unwrap(); + let server = Server::start(&home, Duration::from_secs(3600)); + home.create("acme"); + let admin = connect( + &home, + "acme", + home.token("acme", &["read", "write", "post", "admin"]), + ) + .await; + set_targets(&admin, None, Some(&folder)).await; + post_one(&admin, 2).await; + let before = head(&admin).await; + // A reader sees the standing but can't back up. + let reader = connect(&home, "acme", home.token("acme", &["read"])).await; + assert!(call(&reader, "books.backup", json!({})).await.is_ok()); + let refused = call(&reader, "books.back_up_now", json!({})) + .await + .unwrap_err(); + assert!(refused.contains("admin"), "{refused}"); + // Twice: the second has nothing new to commit, and still answers. + for _ in 0..2 { + let s = call(&admin, "books.back_up_now", json!({})).await.unwrap(); + assert_eq!(s["targets"][0]["through_seq"].as_u64(), Some(before), "{s}"); + assert_eq!(s["targets"][0]["last_error"], Value::Null, "{s}"); + } + assert_eq!(head(&admin).await, before); + assert_eq!( + proven(&home, &folder.join("acme.git"), "now"), + live_head(&home, "acme") + ); + drop((admin, reader)); + assert!(server.close().success()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_backup_that_moved_on_is_never_overwritten() { + let home = Home::new("diverged"); + let remote = home.remote(); + let server = Server::start(&home, Duration::from_secs(3600)); + home.create("acme"); + home.create("other"); + // Another book's history in the remote first. + let other = connect( + &home, + "other", + home.token("other", &["read", "write", "post", "admin"]), + ) + .await; + set_targets(&other, Some(&remote), None).await; + post_one(&other, 9).await; + let s = call(&other, "books.back_up_now", json!({})).await.unwrap(); + assert_eq!(s["targets"][0]["last_error"], Value::Null, "{s}"); + let theirs = proven(&home, &remote, "theirs"); + // This book's push is refused, the remote unchanged. + let acme = connect( + &home, + "acme", + home.token("acme", &["read", "write", "post", "admin"]), + ) + .await; + set_targets(&acme, Some(&remote), None).await; + post_one(&acme, 1).await; + let s = call(&acme, "books.back_up_now", json!({})).await.unwrap(); + assert!(s["targets"][0]["last_error"].is_string(), "{s}"); + assert_eq!(s["targets"][0]["through_seq"], Value::Null, "{s}"); + assert_eq!(proven(&home, &remote, "after"), theirs); + drop((other, acme)); + assert!(server.close().success()); +} + +/// Restore `source` as `id`: the preview, then (if it was one) the +/// restore confirmed against its head. Ok(what was restored) or Err(why). +fn restore(home: &Home, id: &str, source: &Path, replace: bool) -> Result { + let request = |expect_head: Option<(u64, String)>| Request::Restore { + id: id.into(), + source: source.display().to_string(), + replace, + confirmed: expect_head.is_some(), + expect_head, + }; + let preview = match home.admin(request(None)) { + Response::RestorePreview(p) => p, + Response::Refused(why) => return Err(why), + other => panic!("{other:?}"), + }; + // (Whether books of that id exist is said; the restore itself refuses + // over them, or replacing none.) + match home.admin(request(Some(preview.head.clone()))) { + Response::Restored(r) => { + assert_eq!(r.head, preview.head); + assert_eq!(r.problem, None); + Ok(r) + } + Response::Refused(why) => Err(why), + other => panic!("{other:?}"), + } +} + +fn books_held(home: &Home) -> Vec { + match home.admin(Request::List) { + Response::Books(b) => b.into_iter().map(|b| b.id).collect(), + other => panic!("{other:?}"), + } +} + +/// What the books say, for comparing two that should be the same. +async fn figures(c: &Client) -> Value { + json!({ + "tb": call(c, "books.trial_balance", json!({})).await.unwrap(), + "receipts": call(c, "books.list_receipts", json!({})).await.unwrap(), + "settings": call(c, "books.settings", json!({})).await.unwrap()["settings"].clone(), + }) +} + +/// A git command in `dir` with an identity, for making damaged backups. +fn git(dir: &Path, args: &[&str]) { + let out = Command::new("git") + .arg("-C") + .arg(dir) + .args([ + "-c", + "user.name=t", + "-c", + "user.email=t@t.invalid", + "-c", + "commit.gpgsign=false", + ]) + .args(args) + .output() + .unwrap(); + assert!(out.status.success(), "{args:?}: {out:?}"); +} + +/// A backup of acme (two entries, a receipt each) in the folder target. +async fn backed_up_acme(home: &Home) -> (PathBuf, Value) { + settle().await; + let folder = home.outside("disk"); + std::fs::create_dir_all(&folder).unwrap(); + let c = connect( + home, + "acme", + home.token("acme", &["read", "write", "post", "admin"]), + ) + .await; + set_targets(&c, None, Some(&folder)).await; + post_one(&c, 4).await; + post_one(&c, 5).await; + let s = call(&c, "books.back_up_now", json!({})).await.unwrap(); + assert_eq!(s["targets"][0]["last_error"], Value::Null, "{s}"); + (folder.join("acme.git"), figures(&c).await) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn restore_beside_existing_books_and_on_a_clean_install() { + let home = Home::new("beside"); + let server = Server::start(&home, Duration::from_secs(3600)); + home.create("acme"); + let (backup, want) = backed_up_acme(&home).await; + // Beside: its own books, the same figures and head. + let r = restore(&home, "acme-copy", &backup, false).unwrap(); + assert_eq!(r.replaced, None); + assert_eq!(r.head, live_head(&home, "acme")); + let copy = connect( + &home, + "acme-copy", + home.token("acme-copy", &["read", "admin"]), + ) + .await; + assert_eq!(figures(©).await, want); + // Its own backups carry on from the restored history: to the folder + // under its own name. + let s = call(©, "books.back_up_now", json!({})).await.unwrap(); + assert_eq!(s["targets"][0]["last_error"], Value::Null, "{s}"); + let folder = backup.parent().unwrap(); + assert_eq!( + proven(&home, &folder.join("acme-copy.git"), "copy"), + live_head(&home, "acme-copy") + ); + drop(copy); + assert!(server.close().success()); + + // restore_on_clean_install: another computer's root, nothing in it. + let other = Home::new("clean"); + let server = Server::start(&other, Duration::from_secs(3600)); + assert_eq!(books_held(&other), Vec::::new()); + let r = restore(&other, "acme", &backup, false).unwrap(); + assert_eq!(r.head, live_head(&home, "acme")); + let c = connect(&other, "acme", other.token("acme", &["read"])).await; + assert_eq!(figures(&c).await, want); + drop(c); + assert!(server.close().success()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn restore_never_overwrites_and_replacing_keeps_the_old_books() { + let home = Home::new("replace"); + let server = Server::start(&home, Duration::from_secs(3600)); + home.create("acme"); + let (backup, want) = backed_up_acme(&home).await; + let at_backup = live_head(&home, "acme"); + // The books move on past the backup. + let c = connect( + &home, + "acme", + home.token("acme", &["read", "write", "post", "admin"]), + ) + .await; + post_one(&c, 6).await; + let moved_on = live_head(&home, "acme"); + assert!(moved_on.0 > at_backup.0); + // Not over them. + let refused = restore(&home, "acme", &backup, false).unwrap_err(); + assert!(refused.contains("already books called"), "{refused}"); + assert_eq!(live_head(&home, "acme"), moved_on); + // Replacing books that aren't there is refused too. + let refused = restore(&home, "nobody", &backup, true).unwrap_err(); + assert!(refused.contains("no books called"), "{refused}"); + // Replacing: the old books kept whole in legacy/, the backup in place. + let r = restore(&home, "acme", &backup, true).unwrap(); + // The restored books are new to the server: backed up as books just + // opened, unasked. + let fresh = connect(&home, "acme", home.token("acme", &["read"])).await; + let deadline = Instant::now() + Duration::from_secs(30); + loop { + let s = call(&fresh, "books.backup", json!({})).await.unwrap(); + if s["targets"][0]["last_ok_at"].is_string() { + break; + } + assert!( + Instant::now() < deadline, + "the restored books weren't backed up: {s}" + ); + tokio::time::sleep(Duration::from_millis(100)).await; + } + drop(fresh); + let kept = home.root().join("legacy").join(r.replaced.clone().unwrap()); + let old = ledger_store::Snapshot::load(&kept).unwrap(); + assert_eq!((old.journal_seq(), old.head_hash().to_string()), moved_on); + assert_eq!(live_head(&home, "acme"), at_backup); + // The session of before never reaches the old books: refused, or + // (its client opening a new session) the restored ones. + if let Ok(start) = call(&c, "books.start", json!({})).await { + assert_eq!(start["head"]["seq"].as_u64(), Some(at_backup.0), "{start}"); + } + let c = connect( + &home, + "acme", + home.token("acme", &["read", "write", "post", "admin"]), + ) + .await; + assert_eq!(figures(&c).await, want); + // And it is written to and backed up as before (the backup's history + // carries on: a fast-forward). + post_one(&c, 7).await; + let s = call(&c, "books.back_up_now", json!({})).await.unwrap(); + assert_eq!(s["targets"][0]["last_error"], Value::Null, "{s}"); + assert_eq!(proven(&home, &backup, "after"), live_head(&home, "acme")); + drop(c); + assert!(server.close().success()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn restore_refuses_a_damaged_backup_and_writes_nothing() { + let home = Home::new("damaged"); + let server = Server::start(&home, Duration::from_secs(3600)); + home.create("acme"); + let (backup, _) = backed_up_acme(&home).await; + let before = books_held(&home); + let books = home.root().join("books"); + let names = || { + let mut v: Vec = std::fs::read_dir(&books) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + v.sort(); + v + }; + let listed = names(); + // Each damage in a backup of its own, made from the good one. + type Spoil<'a> = (&'a str, &'a dyn Fn(&Path)); + let damage: [Spoil; 4] = [ + ("flipped", &|w: &Path| { + let j = w.join(ledger_store::JOURNAL_FILE); + let text = std::fs::read_to_string(&j).unwrap(); + std::fs::write(&j, text.replacen("entry 4", "entry 9", 1)).unwrap(); + git(w, &["commit", "-qam", &git_subject(w)]); + }), + ("no-blob", &|w: &Path| { + git(w, &["rm", "-rq", "blobs"]); + git(w, &["commit", "-qm", &git_subject(w)]); + }), + ("cut", &|w: &Path| { + let j = w.join(ledger_store::JOURNAL_FILE); + let mut bytes = std::fs::read(&j).unwrap(); + bytes.truncate(bytes.len() - 10); + std::fs::write(&j, bytes).unwrap(); + git(w, &["commit", "-qam", &git_subject(w)]); + }), + ("head", &|w: &Path| { + git( + w, + &["commit", "-q", "--allow-empty", "-m", "seq 99 head 00"], + ); + }), + ]; + for (tag, spoil) in damage { + let work = home.outside(&format!("work-{tag}")); + let out = Command::new("git") + .args(["clone", "-q"]) + .arg(&backup) + .arg(&work) + .output() + .unwrap(); + assert!(out.status.success(), "{out:?}"); + spoil(&work); + let bad = home.outside(&format!("bad-{tag}.git")); + let out = Command::new("git") + .args(["clone", "-q", "--bare"]) + .arg(&work) + .arg(&bad) + .output() + .unwrap(); + assert!(out.status.success(), "{out:?}"); + let why = restore(&home, "acme-bad", &bad, false).unwrap_err(); + assert!( + why.contains("can't be restored") || why.contains("can't be trusted"), + "{tag}: {why}" + ); + assert!( + !why.contains(&home.root().display().to_string()), + "{tag}: {why}" + ); + assert!(!why.contains(&bad.display().to_string()), "{tag}: {why}"); + assert_eq!(books_held(&home), before, "{tag}"); + assert_eq!(names(), listed, "{tag}"); + } + // An address that isn't a backup: said, nothing written. + let why = restore(&home, "acme-bad", &home.outside("nothing-here"), false).unwrap_err(); + assert!(!why.contains("nothing-here"), "{why}"); + assert_eq!(names(), listed); + assert!(server.close().success()); +} + +/// The subject a backup commit of `w`'s journal would have. +fn git_subject(w: &Path) -> String { + let s = ledger_store::Snapshot::load(w); + match s { + Ok(s) => format!("seq {} head {}", s.journal_seq(), s.head_hash()), + // A journal that no longer loads: the subject it had. + Err(_) => { + let out = Command::new("git") + .arg("-C") + .arg(w) + .args(["log", "-1", "--format=%s"]) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + } +} + +/// Wait until every target's backup has been drilled. +async fn drilled(c: &Client) -> Value { + let deadline = Instant::now() + Duration::from_secs(60); + loop { + let s = call(c, "books.backup", json!({})).await.unwrap(); + let targets = s["targets"].as_array().unwrap(); + if !targets.is_empty() && targets.iter().all(|t| t["last_drill_at"].is_string()) { + return s; + } + assert!(Instant::now() < deadline, "no drill: {s}"); + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn opening_the_books_drills_a_backup_never_drilled() { + let home = Home::new("drill"); + let server = Server::start(&home, Duration::from_secs(3600)); + home.create("acme"); + let (backup, _) = backed_up_acme(&home).await; + assert!(server.close().success()); + // Opening the books drills a backup never drilled. + let server = Server::start_with( + &home, + Duration::from_secs(3600), + Some(Duration::from_millis(300)), + ); + let c = connect(&home, "acme", home.token("acme", &["read", "admin"])).await; + let s = drilled(&c).await; + assert_eq!(s["targets"][0]["drill_error"], Value::Null, "{s}"); + let start = call(&c, "books.start", json!({})).await.unwrap()["next_steps"].to_string(); + assert!(!start.contains("books.backup"), "{start}"); + // Its backup was proven where it is: the record names the head. + assert!(proven(&home, &backup, "drilled").0 > 0); + drop(c); + assert!(server.close().success()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn restore_drill_failure_is_flagged() { + let home = Home::new("drill-bad"); + let server = Server::start(&home, Duration::from_secs(3600)); + home.create("acme"); + let (backup, _) = backed_up_acme(&home).await; + assert!(server.close().success()); + // The backup is changed behind the books' back. + let work = home.outside("tamper"); + let out = Command::new("git") + .args(["clone", "-q"]) + .arg(&backup) + .arg(&work) + .output() + .unwrap(); + assert!(out.status.success(), "{out:?}"); + let j = work.join(ledger_store::JOURNAL_FILE); + let text = std::fs::read_to_string(&j).unwrap(); + std::fs::write(&j, text.replacen("entry 4", "entry 9", 1)).unwrap(); + git(&work, &["commit", "-qam", "seq 0 head x"]); + git(&work, &["push", "-q", "origin", "HEAD:main"]); + // Opening the books: nothing is pushed over it, and its first drill + // fails, said in books.backup and books.start. + let server = Server::start_with( + &home, + Duration::from_secs(3600), + Some(Duration::from_millis(300)), + ); + let c = connect(&home, "acme", home.token("acme", &["read", "admin"])).await; + let s = drilled(&c).await; + assert!(s["targets"][0]["drill_error"].is_string(), "{s}"); + assert!(s["targets"][0]["last_error"].is_string(), "{s}"); + let start = call(&c, "books.start", json!({})).await.unwrap()["next_steps"].to_string(); + assert!(start.contains("behind or failing"), "{start}"); + drop(c); + assert!(server.close().success()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_write_waits_for_the_quiet_before_it_is_backed_up() { + let home = Home::new("quiet"); + let server = Server::start(&home, Duration::from_secs(3600)); + home.create("acme"); + backed_up_acme(&home).await; + let c = connect( + &home, + "acme", + home.token("acme", &["read", "write", "post", "admin"]), + ) + .await; + post_one(&c, 8).await; + // Several looks of the loop later, still not: the writing hasn't + // stopped for long enough. + tokio::time::sleep(Duration::from_secs(5)).await; + let s = call(&c, "books.backup", json!({})).await.unwrap(); + let head = s["head_seq"].as_u64().unwrap(); + assert!( + s["targets"][0]["through_seq"].as_u64().unwrap() < head, + "{s}" + ); + assert_eq!(s["targets"][0]["stale"], false, "{s}"); + drop(c); + assert!(server.close().success()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_restore_is_of_the_backup_previewed_only() { + let home = Home::new("previewed"); + let server = Server::start(&home, Duration::from_secs(3600)); + home.create("acme"); + let (backup, _) = backed_up_acme(&home).await; + let request = |expect_head: Option<(u64, String)>, confirmed| Request::Restore { + id: "copy".into(), + source: backup.display().to_string(), + replace: false, + confirmed, + expect_head, + }; + // A preview leaves nothing behind in the books' folder, and clears + // what a restore that never finished left. + let books = home.root().join("books"); + std::fs::create_dir(books.join(".restore-junk")).unwrap(); + let previewed = match home.admin(request(None, false)) { + Response::RestorePreview(p) => p.head, + other => panic!("{other:?}"), + }; + let dotted: Vec<_> = std::fs::read_dir(&books) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .filter(|n| n.starts_with('.')) + .collect(); + assert!(dotted.is_empty(), "{dotted:?}"); + // Confirmed with no head: refused. + match home.admin(request(None, true)) { + Response::Refused(why) => assert!(why.contains("preview first"), "{why}"), + other => panic!("{other:?}"), + } + // The backup moves on after the preview: refused, nothing written. + let c = connect( + &home, + "acme", + home.token("acme", &["read", "write", "post", "admin"]), + ) + .await; + post_one(&c, 9).await; + call(&c, "books.back_up_now", json!({})).await.unwrap(); + match home.admin(request(Some(previewed), true)) { + Response::Refused(why) => assert!(why.contains("isn't the one previewed"), "{why}"), + other => panic!("{other:?}"), + } + assert_eq!(books_held(&home), vec!["acme".to_string()]); + assert!(!home.root().join("books").join("copy").exists()); + drop(c); + assert!(server.close().success()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_restore_brings_the_vault_and_its_key() { + let home = Home::new("vault"); + let server = Server::start(&home, Duration::from_secs(3600)); + home.create("acme"); + let c = connect( + &home, + "acme", + home.token("acme", &["read", "write", "post", "admin"]), + ) + .await; + call( + &c, + "books.add_party", + json!({"name": "Mo", "roles": ["member"]}), + ) + .await + .unwrap(); + call( + &c, + "books.set_tin", + json!({"party": "Mo", "tin": "12-3456789", "tin_type": "ein"}), + ) + .await + .unwrap(); + let (backup, want) = backed_up_acme(&home).await; + drop(c); + assert!(server.close().success()); + let other = Home::new("vault-clean"); + let server = Server::start(&other, Duration::from_secs(3600)); + let request = |expect_head: Option<(u64, String)>| Request::Restore { + id: "acme".into(), + source: backup.display().to_string(), + replace: false, + confirmed: expect_head.is_some(), + expect_head, + }; + let preview = match other.admin(request(None)) { + Response::RestorePreview(p) => p, + other => panic!("{other:?}"), + }; + assert_eq!(preview.vault_files, 1); + let r = match other.admin(request(Some(preview.head))) { + Response::Restored(r) => r, + other => panic!("{other:?}"), + }; + assert_eq!(r.vault_files, 1); + // The record and its key, byte for byte. + let files = |h: &Home, sub: &str| { + let mut v: Vec<(String, Vec)> = std::fs::read_dir(h.book_dir("acme").join(sub)) + .unwrap() + .map(|e| { + let e = e.unwrap(); + ( + e.file_name().to_string_lossy().into_owned(), + std::fs::read(e.path()).unwrap(), + ) + }) + .collect(); + v.sort(); + v + }; + for sub in ["vault", "keys"] { + assert!(!files(&home, sub).is_empty(), "{sub}"); + assert_eq!(files(&other, sub), files(&home, sub), "{sub}"); + } + let c = connect(&other, "acme", other.token("acme", &["read"])).await; + assert_eq!(figures(&c).await, want); + let party = call(&c, "books.party", json!({"party": "Mo"})) + .await + .unwrap(); + assert_eq!(party["party"]["tin"]["in_vault"], true, "{party}"); + drop(c); + assert!(server.close().success()); + + // A backup without the vault's key, or without a TIN record the books + // name, is not whole. + let third = Home::new("vault-short"); + let server = Server::start(&third, Duration::from_secs(3600)); + for (gone, said) in [("keys", "key"), ("vault", "TIN record")] { + let work = home.outside(&format!("no-{gone}")); + let out = Command::new("git") + .args(["clone", "-q"]) + .arg(&backup) + .arg(&work) + .output() + .unwrap(); + assert!(out.status.success(), "{out:?}"); + git(&work, &["rm", "-rq", gone]); + git(&work, &["commit", "-qm", &git_subject(&work)]); + let bad = home.outside(&format!("no-{gone}.git")); + let out = Command::new("git") + .args(["clone", "-q", "--bare"]) + .arg(&work) + .arg(&bad) + .output() + .unwrap(); + assert!(out.status.success(), "{out:?}"); + let why = restore(&third, "acme", &bad, false).unwrap_err(); + assert!(why.contains(said), "{gone}: {why}"); + assert_eq!(books_held(&third), Vec::::new()); + } + assert!(server.close().success()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_replacing_restore_that_fails_leaves_the_books_open() { + let home = Home::new("replace-bad"); + let server = Server::start(&home, Duration::from_secs(3600)); + home.create("acme"); + let (backup, want) = backed_up_acme(&home).await; + let before = live_head(&home, "acme"); + // A backup whose head isn't the one it records. + let work = home.outside("work"); + let out = Command::new("git") + .args(["clone", "-q"]) + .arg(&backup) + .arg(&work) + .output() + .unwrap(); + assert!(out.status.success(), "{out:?}"); + git( + &work, + &["commit", "-q", "--allow-empty", "-m", "seq 99 head 00"], + ); + let bad = home.outside("bad.git"); + let out = Command::new("git") + .args(["clone", "-q", "--bare"]) + .arg(&work) + .arg(&bad) + .output() + .unwrap(); + assert!(out.status.success(), "{out:?}"); + // Its preview is refused; so is a confirm (sent without one), which + // lets the books go first, then fails: they are open again, as they + // were, nothing moved to legacy. + assert!(restore(&home, "acme", &bad, true).is_err()); + let why = match home.admin(Request::Restore { + id: "acme".into(), + source: bad.display().to_string(), + replace: true, + confirmed: true, + expect_head: Some(before.clone()), + }) { + Response::Refused(why) => why, + other => panic!("{other:?}"), + }; + assert!(why.contains("can't be"), "{why}"); + match home.admin(Request::List) { + Response::Books(b) => { + assert_eq!(b.len(), 1); + assert!(b[0].open, "{b:?}"); + } + other => panic!("{other:?}"), + } + assert_eq!(live_head(&home, "acme"), before); + let legacy = home.root().join("legacy"); + assert!(!legacy.exists() || std::fs::read_dir(&legacy).unwrap().count() == 0); + let c = connect( + &home, + "acme", + home.token("acme", &["read", "write", "post", "admin"]), + ) + .await; + assert_eq!(figures(&c).await, want); + post_one(&c, 3).await; + drop(c); + assert!(server.close().success()); +} + +/// The book's backup record (the test's own data root). +fn records(home: &Home, book: &str) -> Vec { + std::fs::read_to_string(home.book_dir(book).join("ops").join("backup.jsonl")) + .unwrap_or_default() + .lines() + .map(|l| serde_json::from_str(l).unwrap()) + .collect() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn books_restored_beside_their_original_leave_its_repository_alone() { + let home = Home::new("beside-remote"); + let server = Server::start(&home, Duration::from_secs(3600)); + home.create("acme"); + settle().await; + let (remote, folder) = (home.remote(), home.outside("disk")); + std::fs::create_dir_all(&folder).unwrap(); + let c = connect( + &home, + "acme", + home.token("acme", &["read", "write", "post", "admin"]), + ) + .await; + set_targets(&c, Some(&remote), Some(&folder)).await; + post_one(&c, 1).await; + let s = call(&c, "books.back_up_now", json!({})).await.unwrap(); + assert!( + s["targets"] + .as_array() + .unwrap() + .iter() + .all(|t| t["last_error"].is_null()), + "{s}" + ); + // A copy beside them, from their repository. + let r = restore(&home, "copy", &remote, false).unwrap(); + assert_eq!(r.origin.as_deref(), Some("acme")); + let copy = connect( + &home, + "copy", + home.token("copy", &["read", "write", "post", "admin"]), + ) + .await; + post_one(©, 2).await; + let s = call(©, "books.back_up_now", json!({})).await.unwrap(); + let by = |kind: &str| { + s["targets"] + .as_array() + .unwrap() + .iter() + .find(|t| t["target"] == kind) + .unwrap() + .clone() + }; + // Its own folder repository; the shared one paused, and said. + assert_eq!(by("folder")["last_error"], Value::Null, "{s}"); + assert!( + by("repository")["last_error"] + .as_str() + .unwrap() + .contains("restored as a copy"), + "{s}" + ); + let start = call(©, "books.start", json!({})).await.unwrap()["next_steps"].to_string(); + assert!(start.contains("behind or failing"), "{start}"); + // The original carries on backing up there. + post_one(&c, 3).await; + let s = call(&c, "books.back_up_now", json!({})).await.unwrap(); + assert!( + s["targets"] + .as_array() + .unwrap() + .iter() + .all(|t| t["last_error"].is_null()), + "{s}" + ); + assert_eq!(proven(&home, &remote, "orig"), live_head(&home, "acme")); + // Naming a repository of its own ends the pause. + let own = home.outside("own.git"); + let out = Command::new("git") + .args(["init", "-q", "--bare", "-b", "main"]) + .arg(&own) + .output() + .unwrap(); + assert!(out.status.success(), "{out:?}"); + set_targets(©, Some(&own), None).await; + let s = call(©, "books.back_up_now", json!({})).await.unwrap(); + assert!( + s["targets"] + .as_array() + .unwrap() + .iter() + .all(|t| t["last_error"].is_null()), + "{s}" + ); + drop((c, copy)); + assert!(server.close().success()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn restored_books_never_make_their_backup_folder_again() { + let home = Home::new("restore-mount"); + let server = Server::start(&home, Duration::from_secs(3600)); + home.create("acme"); + settle().await; + let (remote, folder) = (home.remote(), home.outside("disk")); + std::fs::create_dir_all(&folder).unwrap(); + let c = connect( + &home, + "acme", + home.token("acme", &["read", "write", "post", "admin"]), + ) + .await; + set_targets(&c, Some(&remote), Some(&folder)).await; + post_one(&c, 1).await; + call(&c, "books.back_up_now", json!({})).await.unwrap(); + drop(c); + assert!(server.close().success()); + // On another computer, its backup disk not mounted (the mount point + // there, empty): restored from the repository. + std::fs::rename(folder.join("acme.git"), home.outside("unmounted.git")).unwrap(); + let other = Home::new("restore-mount-new"); + let server = Server::start(&other, Duration::from_secs(3600)); + restore(&other, "acme", &remote, false).unwrap(); + let c = connect(&other, "acme", other.token("acme", &["read", "admin"])).await; + let s = call(&c, "books.back_up_now", json!({})).await.unwrap(); + let by = |kind: &str| { + s["targets"] + .as_array() + .unwrap() + .iter() + .find(|t| t["target"] == kind) + .unwrap() + .clone() + }; + assert!( + by("folder")["last_error"] + .as_str() + .unwrap() + .contains("holds no backup"), + "{s}" + ); + assert!(!folder.join("acme.git").exists(), "made on the wrong disk"); + assert_eq!(by("repository")["last_error"], Value::Null, "{s}"); + // The human says it's a new disk: begun there again. + let s = call(&c, "books.back_up_now", json!({"new_disk": true})) + .await + .unwrap(); + assert!( + s["targets"] + .as_array() + .unwrap() + .iter() + .all(|t| t["last_error"].is_null()), + "{s}" + ); + assert!(folder.join("acme.git").join("HEAD").is_file()); + drop(c); + assert!(server.close().success()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_damaged_backup_record_never_keeps_the_books_shut() { + let home = Home::new("damaged-record"); + let server = Server::start(&home, Duration::from_secs(3600)); + home.create("acme"); + backed_up_acme(&home).await; + assert!(server.close().success()); + let log = home.book_dir("acme").join("ops").join("backup.jsonl"); + let text = std::fs::read_to_string(&log).unwrap(); + std::fs::write(&log, text.replacen("\"ok\":true", "\"ok\":false", 1)).unwrap(); + let server = Server::start(&home, Duration::from_secs(3600)); + match home.admin(Request::List) { + Response::Books(b) => assert!(b[0].open, "{b:?}"), + other => panic!("{other:?}"), + } + let aside: Vec<_> = std::fs::read_dir(home.book_dir("acme").join("ops")) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .filter(|n| n.contains("damaged")) + .collect(); + assert_eq!(aside.len(), 1, "{aside:?}"); + let c = connect(&home, "acme", home.token("acme", &["read", "admin"])).await; + let s = call(&c, "books.back_up_now", json!({})).await.unwrap(); + assert_eq!(s["targets"][0]["last_error"], Value::Null, "{s}"); + // What it knew of whose backups the folder holds was kept: gone, the + // folder's backup is a disk not connected, never made again. + drop(c); + assert!(server.close().success()); + let folder = home.outside("disk"); + std::fs::rename(folder.join("acme.git"), home.outside("away.git")).unwrap(); + let server = Server::start(&home, Duration::from_secs(3600)); + let c = connect(&home, "acme", home.token("acme", &["read", "admin"])).await; + let s = call(&c, "books.back_up_now", json!({})).await.unwrap(); + assert!( + s["targets"][0]["last_error"] + .as_str() + .unwrap() + .contains("holds no backup"), + "{s}" + ); + assert!(!folder.join("acme.git").exists()); + drop(c); + assert!(server.close().success()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_failing_target_waits_while_the_others_carry_on() { + let home = Home::new("one-failing"); + let server = Server::start(&home, Duration::from_millis(300)); + home.create("acme"); + settle().await; + let remote = home.remote(); + let unplugged = home.outside("unplugged"); + let c = connect( + &home, + "acme", + home.token("acme", &["read", "write", "post", "admin"]), + ) + .await; + set_targets(&c, Some(&remote), Some(&unplugged)).await; + let s = call(&c, "books.back_up_now", json!({})).await.unwrap(); + let folder_fails = |home: &Home| { + records(home, "acme") + .iter() + .filter(|r| r["kind"] == "push" && r["target"] == "folder") + .count() + }; + let before = folder_fails(&home); + assert!(before >= 1, "{s}"); + // Writes: backed up to the repository soon after each, the folder not + // tried again (it failed lately). + for n in 0..3 { + post_one(&c, 10 + n).await; + let deadline = Instant::now() + Duration::from_secs(30); + loop { + let s = call(&c, "books.backup", json!({})).await.unwrap(); + let remote_t = s["targets"] + .as_array() + .unwrap() + .iter() + .find(|t| t["target"] == "repository") + .unwrap() + .clone(); + if remote_t["through_seq"] == s["head_seq"] { + break; + } + assert!(Instant::now() < deadline, "not backed up: {s}"); + tokio::time::sleep(Duration::from_millis(100)).await; + } + } + assert_eq!(folder_fails(&home), before); + drop(c); + assert!(server.close().success()); +} diff --git a/crates/ledger-cli/tests/open_while_held.rs b/crates/ledger-cli/tests/open_while_held.rs index 75d0d7e..c358713 100644 --- a/crates/ledger-cli/tests/open_while_held.rs +++ b/crates/ledger-cli/tests/open_while_held.rs @@ -19,8 +19,7 @@ use std::time::{Duration, Instant}; /// Short timings for tests (the defaults are the plan's), and agents' /// tokens kept in files in the test root, not this computer's keyring. -const KNOBS: &str = - "close_after=2500,backoff=50,exe_poll=100,init_wait=5000,queue_wait=30000,file_keyring=1"; +const KNOBS: &str = "close_after=2500,backoff=50,exe_poll=100,init_wait=5000,queue_wait=30000,file_keyring=1,backup_quiet=300"; /// A home with a data root, and the program the tests run (possibly a /// copy, for restarts). @@ -921,17 +920,13 @@ fn the_lease_socket_takes_only_the_owners_intents() { "{\"proto\":1,\"intent\":{\"consent_format_upgrade\":{\"book\":\"acme\"}}}\n", "passphrase", ), - ( - "{\"proto\":1,\"intent\":{\"restore\":{\"staged\":\"s1\",\"replace\":\"acme\"}}}\n", - "passphrase", - ), ( "{\"proto\":1,\"intent\":{\"back_up_now\":{\"book\":\"acme\"}}}\n", - "isn't built yet", + "books.back_up_now", ), ( - "{\"proto\":1,\"intent\":{\"restore\":{\"staged\":\"s1\"}}}\n", - "isn't built yet", + "{\"proto\":1,\"intent\":{\"restore\":{\"source\":\"/x\",\"id\":\"acme\",\"replace\":true}}}\n", + "wanderbooks books restore", ), ( "{\"proto\":1,\"intent\":{\"new_books\":{\"id\":\"beta\",\"currency\":\"USD\"}}}\n", @@ -2346,3 +2341,126 @@ fn the_commands_given_run_as_the_agents_own_token() { assert!(String::from_utf8_lossy(&out.stderr).contains("isn't UTF-8")); a.end(); } + +#[test] +fn a_backup_is_restored_from_the_command_line() { + let home = Home::new("restore"); + home.create("acme"); + let disk = home.dir.path().join("disk"); + std::fs::create_dir(&disk).unwrap(); + let mut a = Agent::start(&home, "acme"); + a.initialize(); + chart(&mut a); + post_one(&mut a, 1).unwrap(); + a.call( + "books.set_setting", + json!({"changes": [{"key": "backup.folder", "value": disk.display().to_string()}], + "decided_by": "Test Owner"}), + ) + .unwrap(); + // Backed up by the server alone, soon after (the host passed on its + // quiet). + let mut s = Value::Null; + assert!( + wait_for(30, || { + s = a.call("books.backup", json!({})).unwrap(); + s["targets"][0]["through_seq"] == s["head_seq"] + }), + "never backed up: {s}" + ); + assert_eq!(s["targets"][0]["last_error"], Value::Null, "{s}"); + let head = s["head_seq"].as_u64().unwrap(); + let backup = disk.join("acme.git"); + let backup = backup.to_str().unwrap(); + // Previewed, and not done without a yes. + let out = home.books(&["restore", backup, "--id", "copy"]); + let said = String::from_utf8_lossy(&out.stdout); + assert!(said.contains(&format!("through event {head}")), "{out:?}"); + assert!( + String::from_utf8_lossy(&out.stderr).contains("confirm at a terminal"), + "{out:?}" + ); + assert_eq!(out.status.code(), Some(1)); + // Under its own name, over books that exist: refused. + let out = home.books(&["restore", backup, "--yes"]); + assert!( + String::from_utf8_lossy(&out.stderr).contains("already books called"), + "{out:?}" + ); + // A repository named otherwise: restored as the books it holds, by + // default (here: over those, refused), said. + let renamed = home.dir.path().join("disk").join("fernhill-backup.git"); + let out = Command::new("git") + .args(["clone", "-q", "--bare"]) + .arg(backup) + .arg(&renamed) + .output() + .unwrap(); + assert!(out.status.success(), "{out:?}"); + let out = home.books(&["restore", renamed.to_str().unwrap(), "--yes"]); + assert!( + String::from_utf8_lossy(&out.stderr).contains("already books called \"acme\""), + "{out:?}" + ); + let out = home.books(&["restore", renamed.to_str().unwrap(), "--id", "other"]); + let said = String::from_utf8_lossy(&out.stdout); + assert!( + said.contains("It is a backup of the books \"acme\""), + "{out:?}" + ); + assert!(said.contains("--id acme"), "{out:?}"); + // A repository whose name is no id: still restored as its books. + let odd = home.dir.path().join("disk").join("Fernhill_Backup.git"); + std::fs::rename(&renamed, &odd).unwrap(); + let out = home.books(&["restore", odd.to_str().unwrap(), "--yes"]); + assert!( + String::from_utf8_lossy(&out.stderr).contains("already books called \"acme\""), + "{out:?}" + ); + std::fs::rename(&odd, &renamed).unwrap(); + // Beside them. + let out = home.books(&["restore", backup, "--id", "copy", "--yes"]); + assert!(out.status.success(), "{out:?}"); + assert!( + String::from_utf8_lossy(&out.stdout).contains("Restored as \"copy\""), + "{out:?}" + ); + // Replacing them, the old kept: from the repository named otherwise, + // given no id (the books it holds are replaced). + post_one(&mut a, 2).unwrap(); + let out = home.books(&["restore", renamed.to_str().unwrap(), "--replace", "--yes"]); + assert!(out.status.success(), "{out:?}"); + let said = String::from_utf8_lossy(&out.stdout); + assert!( + said.contains("kept in the legacy folder as acme-"), + "{out:?}" + ); + // The head it restored (post 2 may have been backed up by then). + let restored: u64 = said + .split("Restored as \"acme\": through event ") + .nth(1) + .and_then(|rest| rest.split(' ').next()) + .and_then(|n| n.parse().ok()) + .expect("the restored head is said"); + assert!(restored >= head, "{said}"); + let head = restored; + assert!(!said.contains(&home.root().display().to_string()), "{said}"); + let out = home.books(&["list"]); + let listed = String::from_utf8_lossy(&out.stdout); + assert!( + listed.contains("acme open") && listed.contains("copy open"), + "{out:?}" + ); + // The session of before never reaches the books replaced: refused, or + // (its bridge opening a new session) the restored books. + if let Ok(start) = a.call("books.start", json!({})) { + assert_eq!(start["head"]["seq"].as_u64(), Some(head), "{start}"); + } + a.end(); + // A new session sees the restored books, at the backup's head. + let mut fresh = Agent::start(&home, "acme"); + fresh.initialize(); + let start = fresh.call("books.start", json!({})).unwrap(); + assert_eq!(start["head"]["seq"].as_u64(), Some(head), "{start}"); + fresh.end(); +} diff --git a/crates/ledger-core/src/settings.rs b/crates/ledger-core/src/settings.rs index 77d6628..daa0f24 100644 --- a/crates/ledger-core/src/settings.rs +++ b/crates/ledger-core/src/settings.rs @@ -122,6 +122,12 @@ pub enum Kind { NameControl, /// An asset tag prefix: one to eight letters or digits. TagPrefix, + /// A repository's address, as git takes it: one line, never one that + /// reads as an option. + Repository, + /// A folder named from the root of its disk: one line, an absolute + /// path. + Folder, } /// One setting the books know. @@ -360,6 +366,20 @@ pub static KEYS: &[Key] = &[ sensitivity: Sensitivity::Public, doc: "off: approval_rule conventions are recorded but never block a post. enforce: an entry a rule covers posts only once its approver approves it (books.approve_entry). Unset: off.", }, + Key { + name: "backup.git_remote", + toml: ("backup", "git_remote"), + kind: Kind::Repository, + sensitivity: Sensitivity::Public, + doc: "A private repository the books back themselves up to (plan D-30): its address as its host shows it for cloning, over ssh or https (such as https://host/owner/books), or a path on this computer. Unset: none.", + }, + Key { + name: "backup.folder", + toml: ("backup", "folder"), + kind: Kind::Folder, + sensitivity: Sensitivity::Public, + doc: "A folder the books back themselves up to as well (D-30): an absolute path, an external disk or a synced folder; the backup is a repository in it named for the book. Unset: none.", + }, ]; /// The key named `name`. @@ -873,6 +893,37 @@ pub fn check(key: &Key, v: &Value) -> Result { } Ok(json!(s)) } + Kind::Repository => { + let s = line_text(name, want_str(name, v)?, MAX_TEXT)?; + if s.starts_with('-') { + return Err(format!( + "{name} {s:?}: a repository's address never starts with -" + )); + } + // A path on this computer is named from the root of its disk: + // the server's working folder is no place to name one from. + let networked = match s.split_once("://") { + Some((scheme, _)) => scheme != "file", + None => s + .split_once(':') + .is_some_and(|(host, _)| !host.contains('/')), + }; + if !networked && !s.starts_with("file://") && !std::path::Path::new(&s).is_absolute() { + return Err(format!( + "{name} {s:?}: a repository on this computer is named from the root of its disk (such as /srv/books)" + )); + } + Ok(json!(s)) + } + Kind::Folder => { + let s = line_text(name, want_str(name, v)?, MAX_TEXT)?; + if !std::path::Path::new(&s).is_absolute() { + return Err(format!( + "{name} {s:?}: name the folder from the root of its disk (such as /media/backup)" + )); + } + Ok(json!(s)) + } } } @@ -1243,6 +1294,16 @@ impl Settings { } } + /// `backup.git_remote`, if set. + pub fn backup_git_remote(&self) -> Option<&str> { + self.str("backup.git_remote") + } + + /// `backup.folder`, if set. + pub fn backup_folder(&self) -> Option<&str> { + self.str("backup.folder") + } + /// `company.timezone`, if set. pub fn timezone(&self) -> Option<&str> { self.str("company.timezone") @@ -1423,6 +1484,44 @@ pub fn display(key: &Key, v: &Value, full: bool) -> String { mod tests { use super::*; + #[test] + fn where_backups_go_is_checked() { + let remote = key("backup.git_remote").unwrap(); + let folder = key("backup.folder").unwrap(); + for ok in [ + "git@host:owner/books.git", + "https://host/owner/books", + "/srv/books.git", + ] { + assert_eq!(check(remote, &json!(ok)).unwrap(), json!(ok), "{ok}"); + } + assert_eq!( + check(remote, &json!(" ssh://h/b ")).unwrap(), + json!("ssh://h/b") + ); + for bad in [ + "--upload-pack=x", + "-", + "", + "a\nb", + "acme.git", + "backups/acme.git", + "~/x.git", + "./x", + ] { + assert!(check(remote, &json!(bad)).is_err(), "{bad}"); + } + assert!(check(remote, &json!("file:///srv/b.git")).is_ok()); + #[cfg(unix)] + assert_eq!( + check(folder, &json!("/media/backup")).unwrap(), + json!("/media/backup") + ); + for bad in ["backup", "./backup", "~/backup", "-x", ""] { + assert!(check(folder, &json!(bad)).is_err(), "{bad}"); + } + } + fn src() -> Source { Source { decided_by: "Owner".into(), diff --git a/crates/ledger-host/src/commands.rs b/crates/ledger-host/src/commands.rs index 14c6aab..084e23c 100644 --- a/crates/ledger-host/src/commands.rs +++ b/crates/ledger-host/src/commands.rs @@ -14,9 +14,9 @@ //! 1. A command that needs the server holds a lease while it runs, and //! lets go when it ends. //! 2. Nothing printed names the data root. -//! 3. Adoption is previewed first and goes ahead only on a yes at the -//! terminal, or --yes; a vault beside the books must be named (or -//! --no-vault given): it is never silently left behind. +//! 3. Adoption and restoring are previewed first and go ahead only on a +//! yes at the terminal, or --yes; a vault beside the books must be +//! named (or --no-vault given): it is never silently left behind. //! 4. A token made is kept in the keyring before anything is said of it, //! and its text is printed only by `token print`; one that can't be //! kept is revoked again. A token revoked is forgotten by the keyring @@ -392,12 +392,16 @@ fn books_usage(prog: &str) { eprintln!( " {prog} adopt DATA-DIR --id ID [--toml FILE] [--config-json FILE] [--vault DIR --vault-key FILE | --no-vault] [--skip-unknown] [--yes]" ); + eprintln!(" {prog} restore BACKUP [--id ID] [--replace] [--yes]"); eprintln!(" list the books, and whether each is open"); eprintln!(" create start new, empty books (base currency USD unless given)"); eprintln!(" adopt copy existing books in, verified; the original is left as it was"); + eprintln!( + " restore bring back a backup (a git repository's address, or BOOK.git in the backup folder), proven whole first: as new books, or replacing books (kept in legacy/)" + ); } -/// `wanderbooks books list|create|adopt`. +/// `wanderbooks books list|create|adopt|restore`. pub fn books( prog: &str, args: impl IntoIterator, @@ -413,6 +417,7 @@ pub fn books( let mut no_vault = false; let mut skip_unknown = false; let mut yes = false; + let mut replace = false; let mut args = args.into_iter(); while let Some(arg) = args.next() { if common.take(&arg, &mut args)? { @@ -428,6 +433,7 @@ pub fn books( "--vault-key" => vault_key = Some(PathBuf::from(value("a file")?)), "--no-vault" => no_vault = true, "--skip-unknown" => skip_unknown = true, + "--replace" => replace = true, "--yes" => yes = true, "--help" | "-h" => { books_usage(prog); @@ -471,6 +477,7 @@ pub fn books( yes, ); } + ["restore", source] => return restore(&common, source, id, replace, yes), _ => { books_usage(prog); return Err(exit(2)); @@ -514,6 +521,172 @@ pub fn books( } } +/// The id a backup is restored as when none is given: its repository's +/// name (`acme.git` → `acme`). +fn restore_id(source: &str) -> Option { + let last = source + .trim_end_matches('/') + .rsplit(['/', ':']) + .next()? + .trim_end_matches(".git"); + ledger_store::root::BookId::parse(last) + .ok() + .map(|id| id.to_string()) +} + +/// Ask at the terminal (or take --yes): whether to go ahead. +fn confirmed(question: &str, yes: bool) -> Result { + if yes { + return Ok(true); + } + if !at_a_terminal() { + return Err("confirm at a terminal, or run again with --yes".into()); + } + print!("{question} [y/N] "); + use std::io::Write; + let _ = std::io::stdout().flush(); + let mut answer = String::new(); + std::io::stdin() + .read_line(&mut answer) + .map_err(|e| e.to_string())?; + Ok(matches!(answer.trim(), "y" | "Y" | "yes" | "Yes")) +} + +/// `wanderbooks books restore` (INVARIANT 3). +fn restore( + common: &Common, + source: &str, + id: Option, + replace: bool, + yes: bool, +) -> Result<(), Box> { + let fail = |why: String| -> Result<(), Box> { + eprintln!("error: {why}"); + Err(exit(1)) + }; + // A folder is named to the server by its full path (the server's + // working folder isn't this one); anything else is an address. + let source = match Path::new(source) { + p if p.exists() => std::path::absolute(p) + .unwrap_or_else(|_| p.to_path_buf()) + .display() + .to_string(), + _ => source.to_string(), + }; + // One preview (under the id given, or any: it only reports), then the + // id: given, or else the id of the books the backup holds, or else the + // repository's name. + let probe = id + .clone() + .or_else(|| restore_id(&source)) + .unwrap_or_else(|| "restored".to_string()); + let preview = match with_server( + common, + Request::Restore { + id: probe, + source: source.clone(), + replace, + confirmed: false, + expect_head: None, + }, + ) { + Ok(Response::RestorePreview(p)) => p, + Ok(Response::Refused(why)) | Err(why) => return fail(format!("{source}: {why}")), + Ok(other) => return fail(format!("the books server answered oddly: {other:?}")), + }; + let Some(id) = id + .or_else(|| { + preview + .origin + .clone() + .filter(|o| ledger_store::root::BookId::parse(o).is_ok()) + }) + .or_else(|| restore_id(&source)) + else { + eprintln!("error: name the restored books with --id ID"); + return Err(exit(2)); + }; + let exists = match with_server(common, Request::List) { + Ok(Response::Books(books)) => books.iter().any(|b| b.id == id), + Ok(Response::Refused(why)) | Err(why) => return fail(why), + Ok(other) => return fail(format!("the books server answered oddly: {other:?}")), + }; + if exists && !replace { + return fail(format!( + "there are already books called \"{id}\": restore them under another id (--id), or replace them (--replace; they are kept in legacy/)" + )); + } + if !exists && replace { + return fail(format!( + "there are no books called \"{id}\" to replace: nothing was restored" + )); + } + let request = |expect_head: Option<(u64, String)>| Request::Restore { + id: id.clone(), + source: source.clone(), + replace, + confirmed: expect_head.is_some(), + expect_head, + }; + println!( + "The backup at {source} is whole: through event {} ({}), {} stored files re-hashed, {} TIN vault records.", + preview.head.0, + short_hash(&preview.head.1), + preview.blobs, + preview.vault_files + ); + if let Some(origin) = &preview.origin { + println!("It is a backup of the books \"{origin}\"."); + } + if replace { + println!( + "It replaces the books \"{id}\": they are closed and kept whole in the legacy folder, not deleted." + ); + } else { + println!("It comes back as new books \"{id}\", beside any others."); + } + match preview.origin.as_deref() { + Some(o) if o == id => {} + Some(o) => println!( + "Under a name other than its books' (\"{o}\"), they won't back up to the private repository those books back up to (that would stop theirs): if these are those books, restore them as \"{o}\" (--id {o}); else name a repository of their own." + ), + None => println!( + "It doesn't say whose books it holds: they won't back up to the private repository their settings name until one of their own is named." + ), + } + match confirmed("Restore it?", yes) { + Ok(true) => {} + Ok(false) => { + println!("Not restored."); + return Ok(()); + } + Err(why) => return fail(format!("not restored: {why}")), + } + match with_server(common, request(Some(preview.head.clone()))) { + Ok(Response::Restored(r)) => { + println!( + "Restored as \"{}\": through event {} ({}), {} stored files checked, {} TIN vault records.", + r.id, + r.head.0, + short_hash(&r.head.1), + r.blobs_verified, + r.vault_files + ); + if let Some(kept) = &r.replaced { + println!("The books it replaced are kept in the legacy folder as {kept}."); + } + if let Some(why) = r.problem { + return fail(format!( + "the restored books didn't open: {why}. Closing the books and opening them again tries again." + )); + } + Ok(()) + } + Ok(Response::Refused(why)) | Err(why) => fail(format!("{source}: {why}")), + Ok(other) => fail(format!("the books server answered oddly: {other:?}")), + } +} + /// The files an adoption brings in with the books. struct AdoptFiles { toml: Option, @@ -1743,6 +1916,23 @@ mod tests { } } + #[test] + fn a_backup_is_restored_under_its_repository_name() { + for (source, want) in [ + ("/media/disk/backups/acme.git", Some("acme")), + ("/media/disk/backups/acme.git/", Some("acme")), + ("git@host:owner/acme-books.git", Some("acme-books")), + ("ssh://host/owner/acme", Some("acme")), + ("https://host/owner/acme.git", Some("acme")), + ("acme.git", Some("acme")), + ("/media/disk/Backups/Acme.git", None), + ("/", None), + ("", None), + ] { + assert_eq!(restore_id(source).as_deref(), want, "{source}"); + } + } + #[test] fn an_adoption_says_what_it_did_and_only_that() { let plain = adopted_lines(&adopted()); diff --git a/crates/ledger-host/src/helper.rs b/crates/ledger-host/src/helper.rs index 059a10b..5af7ee1 100644 --- a/crates/ledger-host/src/helper.rs +++ b/crates/ledger-host/src/helper.rs @@ -162,7 +162,9 @@ fn answer(ask: HelperAsk, root: Option<&Path>, knobs: &Knobs) -> Answer { HelperAsk::OpenWith { .. } => { Answer::Refused("opening documents comes with the viewer's menu".into()) } - HelperAsk::StageBackup => Answer::Refused("restoring isn't built yet".into()), + HelperAsk::StageBackup => Answer::Refused( + "restore from a terminal for now: `wanderbooks books restore BACKUP`".into(), + ), HelperAsk::StartUpdater => Answer::Refused("updates come with the installers".into()), } } diff --git a/crates/ledger-host/src/host.rs b/crates/ledger-host/src/host.rs index 19060b9..d493055 100644 --- a/crates/ledger-host/src/host.rs +++ b/crates/ledger-host/src/host.rs @@ -657,6 +657,15 @@ impl Host { if self.explicit_root { command.arg("--root").arg(self.root.path()); } + let d = Knobs::default(); + for (flag, now, default) in [ + ("--backup-quiet", self.knobs.backup_quiet, d.backup_quiet), + ("--drill-first", self.knobs.drill_first, d.drill_first), + ] { + if now != default { + command.arg(flag).arg(now.as_millis().to_string()); + } + } let mut child = command .stdin(std::process::Stdio::piped()) .stdout(std::process::Stdio::piped()) @@ -701,11 +710,16 @@ impl Host { None, ), Intent::BackUpNow { .. } => ( - HostMsg::Refused("backing up isn't built yet (it comes later in P2)".into()), + HostMsg::Refused( + "the books back themselves up; to back up now, ask your agent (books.back_up_now)" + .into(), + ), None, ), Intent::Restore { .. } => ( - HostMsg::Refused("restoring isn't built yet (it comes later in P2)".into()), + HostMsg::Refused( + "restore from a terminal for now: `wanderbooks books restore BACKUP`".into(), + ), None, ), Intent::ConsentFormatUpgrade { .. } => ( diff --git a/crates/ledger-host/src/knobs.rs b/crates/ledger-host/src/knobs.rs index 9bf09ce..80c2b1f 100644 --- a/crates/ledger-host/src/knobs.rs +++ b/crates/ledger-host/src/knobs.rs @@ -30,6 +30,12 @@ pub struct Knobs { /// Keep agents' tokens in files in the root's `run/`, not the OS /// keyring (tests: tokens.rs INVARIANT 3). `file_keyring=1`. pub file_keyring: bool, + /// How long the books stay unwritten before the server backs them up + /// (plan 3.7: 30 s), passed to the server as `--backup-quiet`. + pub backup_quiet: Duration, + /// How long after books open the server first looks for a drill due + /// (60 s), passed to the server as `--drill-first`. + pub drill_first: Duration, } impl Default for Knobs { @@ -43,6 +49,8 @@ impl Default for Knobs { queue_wait: Duration::from_secs(60), open_wait: Duration::from_secs(120), file_keyring: false, + backup_quiet: Duration::from_secs(30), + drill_first: Duration::from_secs(60), } } } @@ -78,6 +86,8 @@ impl Knobs { "init_wait" => k.init_wait = d, "queue_wait" => k.queue_wait = d, "open_wait" => k.open_wait = d, + "backup_quiet" => k.backup_quiet = d, + "drill_first" => k.drill_first = d, other => return Err(format!("{FLAG}: there is no knob {other:?}")), } } @@ -97,6 +107,8 @@ impl Knobs { ("init_wait", self.init_wait, d.init_wait), ("queue_wait", self.queue_wait, d.queue_wait), ("open_wait", self.open_wait, d.open_wait), + ("backup_quiet", self.backup_quiet, d.backup_quiet), + ("drill_first", self.drill_first, d.drill_first), ] { if now != default { parts.push(format!("{name}={}", now.as_millis())); @@ -131,6 +143,8 @@ mod tests { assert_eq!(k.exe_poll, Duration::from_secs(5)); assert_eq!(k.server_delay, Duration::ZERO); assert_eq!(k.open_wait, Duration::from_secs(120)); + assert_eq!(k.backup_quiet, Duration::from_secs(30)); + assert_eq!(k.drill_first, Duration::from_secs(60)); assert_eq!(k.to_arg(), None); assert!(k.args().is_empty()); assert_eq!(Knobs::parse("").unwrap(), k); @@ -139,7 +153,7 @@ mod tests { #[test] fn knobs_pass_on_exactly() { let k = Knobs::parse( - "close_after=500,backoff=20,server_delay=3000,init_wait=100,queue_wait=900,exe_poll=50,open_wait=70", + "close_after=500,backoff=20,server_delay=3000,init_wait=100,queue_wait=900,exe_poll=50,open_wait=70,backup_quiet=40,drill_first=50", ) .unwrap(); assert_eq!(k.close_after, Duration::from_millis(500)); @@ -149,6 +163,8 @@ mod tests { assert_eq!(k.queue_wait, Duration::from_millis(900)); assert_eq!(k.exe_poll, Duration::from_millis(50)); assert_eq!(k.open_wait, Duration::from_millis(70)); + assert_eq!(k.backup_quiet, Duration::from_millis(40)); + assert_eq!(k.drill_first, Duration::from_millis(50)); let args = k.args(); assert_eq!(args[0], FLAG); assert_eq!(Knobs::parse(&args[1]).unwrap(), k); diff --git a/crates/ledger-host/src/lease.rs b/crates/ledger-host/src/lease.rs index ed1913a..cb26f9a 100644 --- a/crates/ledger-host/src/lease.rs +++ b/crates/ledger-host/src/lease.rs @@ -5,7 +5,8 @@ //! the server, through one socket that accepts nothing else. //! SOURCE: plan/served.md 3.9 ("Who holds them", "Three local channels": //! leases record their kind and PID; exactly five owner intents; -//! consent and a replacing restore carry the owner passphrase; +//! consent carries the owner passphrase (a replacing restore no +//! longer does: D-31, the books it replaces are kept in legacy/); //! "Closing": the host says why on each lease; protocol versions N //! and N-1). //! INVARIANTS: @@ -13,11 +14,10 @@ //! connection (the lease lasts until it closes), anything else is //! answered once and the connection closed. //! 2. Exactly five intents exist: close, back_up_now, new_books, -//! restore (a staged backup's id, never a path) and -//! consent_format_upgrade; any other request is refused. -//! 3. consent_format_upgrade, and a restore that replaces a book, carry -//! the owner passphrase; without one they are refused before -//! anything is done. +//! restore (a backup's address or folder, as `books restore` takes +//! it) and consent_format_upgrade; any other request is refused. +//! 3. consent_format_upgrade carries the owner passphrase; without one +//! it is refused before anything is done. //! 4. Before the host ends a lease it sends the reason; a lease that //! ends with no reason means the host is gone. //! 5. Each side accepts its own protocol version and the one before. @@ -120,16 +120,17 @@ pub enum Intent { /// Their base currency. currency: String, }, - /// Restore a staged backup, as its own book or replacing one. + /// Restore a backup, as its own book or replacing one (kept in + /// legacy/). Restore { - /// The staged backup's id (never a path). - staged: String, - /// The book it replaces, if it replaces one. + /// The backup: a repository's address, or its folder (`.git` + /// in the backup folder). + source: String, + /// The books' id. + id: String, + /// Replace the books of that id. #[serde(default)] - replace: Option, - /// The owner passphrase: needed to replace (INVARIANT 3). - #[serde(default)] - passphrase: Option, + replace: bool, }, /// Let the server upgrade a book's journal format. ConsentFormatUpgrade { @@ -151,13 +152,6 @@ impl Intent { "consenting to a format upgrade needs the owner passphrase, typed in the app" .into(), ), - Intent::Restore { - replace: Some(_), - passphrase, - .. - } if missing(passphrase) => Err( - "a restore that replaces books needs the owner passphrase, typed in the app".into(), - ), _ => Ok(()), } } @@ -288,7 +282,7 @@ mod tests { json!("close"), json!({"back_up_now": {"book": "acme"}}), json!({"new_books": {"id": "acme", "currency": "USD"}}), - json!({"restore": {"staged": "s1"}}), + json!({"restore": {"source": "/backups/acme.git", "id": "acme"}}), json!({"consent_format_upgrade": {"book": "acme", "passphrase": "x"}}), ] { let h = parse(json!({"proto": 1, "intent": intent})).unwrap(); @@ -297,7 +291,8 @@ mod tests { for bad in [ json!({"proto": 1, "intent": "delete"}), json!({"proto": 1, "intent": {"restore": {"path": "/x"}}}), - json!({"proto": 1, "intent": {"restore": {"staged": "s1", "path": "/x"}}}), + json!({"proto": 1, "intent": {"restore": {"source": "/x", "id": "a", "path": "/x"}}}), + json!({"proto": 1, "intent": {"restore": {"staged": "s1"}}}), json!({"proto": 1, "intent": {"run": {"command": "sh"}}}), json!({"proto": 1, "exec": "sh"}), json!({"proto": 1}), @@ -322,7 +317,7 @@ mod tests { } #[test] - fn consent_and_a_replacing_restore_need_the_passphrase() { + fn only_consent_needs_the_passphrase() { let consent = |p: Option<&str>| Intent::ConsentFormatUpgrade { book: "acme".into(), passphrase: p.map(str::to_string), @@ -330,16 +325,13 @@ mod tests { assert!(consent(None).check_passphrase().is_err()); assert!(consent(Some("")).check_passphrase().is_err()); assert!(consent(Some("pw")).check_passphrase().is_ok()); - let restore = |replace: Option<&str>, p: Option<&str>| Intent::Restore { - staged: "s1".into(), - replace: replace.map(str::to_string), - passphrase: p.map(str::to_string), - }; - assert!(restore(None, None).check_passphrase().is_ok()); - assert!(restore(Some("acme"), None).check_passphrase().is_err()); - assert!(restore(Some("acme"), Some("")).check_passphrase().is_err()); - assert!(restore(Some("acme"), Some("pw")).check_passphrase().is_ok()); for other in [ + // D-31: replacing keeps the old books in legacy/; no passphrase. + Intent::Restore { + source: "/backups/acme.git".into(), + id: "acme".into(), + replace: true, + }, Intent::Close, Intent::BackUpNow { book: "a".into() }, Intent::NewBooks { diff --git a/crates/ledger-host/src/start.rs b/crates/ledger-host/src/start.rs index b15b6a1..855dc32 100644 --- a/crates/ledger-host/src/start.rs +++ b/crates/ledger-host/src/start.rs @@ -9,8 +9,11 @@ //! 1. The host's environment is a fixed list: PATH (fixed), HOME (from //! the account's record, never the opener's), and XDG_RUNTIME_DIR //! and DBUS_SESSION_BUS_ADDRESS only when they point at this -//! account's own private runtime folder. Nothing else passes (a -//! scope's own identifiers, such as INVOCATION_ID, are systemd's). +//! account's own private runtime folder, and SSH_AUTH_SOCK only when +//! it names a socket this account owns (the owner's ssh agent, so the +//! server's backups reach a private git repository, D-30). Nothing +//! else passes (a scope's own identifiers, such as INVOCATION_ID, are +//! systemd's). //! 2. Its standard input and output are null, its errors go to //! run/host.log (0600); it starts a new session with umask 077. Every //! other descriptor is closed (std opens them close-on-exec, and the @@ -112,8 +115,20 @@ fn session_bus(runtime: &Path) -> Option { } } -/// The host's environment (INVARIANT 1). -fn host_env(home: &Path) -> Vec<(String, std::ffi::OsString)> { +/// The ssh agent's socket named in `named`, if it is a socket this +/// account owns at an absolute path (INVARIANT 1). +fn own_agent(named: Option<&std::ffi::OsStr>) -> Option { + use std::os::unix::fs::FileTypeExt; + let path = Path::new(named?); + (path.is_absolute() + && std::fs::symlink_metadata(path) + .is_ok_and(|m| m.file_type().is_socket() && m.uid() == own_uid())) + .then(|| path.as_os_str().to_os_string()) +} + +/// The host's environment (INVARIANT 1), given the opener's +/// `SSH_AUTH_SOCK`. +fn host_env(home: &Path, ssh_agent: Option<&std::ffi::OsStr>) -> Vec<(String, std::ffi::OsString)> { let mut env = vec![ ("PATH".to_string(), PATH.into()), ("HOME".to_string(), home.as_os_str().to_os_string()), @@ -124,6 +139,9 @@ fn host_env(home: &Path) -> Vec<(String, std::ffi::OsString)> { } env.push(("XDG_RUNTIME_DIR".into(), runtime.into_os_string())); } + if let Some(agent) = own_agent(ssh_agent) { + env.push(("SSH_AUTH_SOCK".into(), agent)); + } env } @@ -199,7 +217,10 @@ pub fn start_host(root: Option<&Path>, knobs: &Knobs) -> Result<(), String> { }; c.args(&args) .env_clear() - .envs(host_env(&home)) + .envs(host_env( + &home, + std::env::var_os("SSH_AUTH_SOCK").as_deref(), + )) .current_dir("/") .stdin(Stdio::null()) .stdout(Stdio::null()) @@ -357,12 +378,17 @@ mod tests { #[test] fn the_host_environment_carries_nothing_else() { - let env = host_env(Path::new("/home/someone")); + let env = host_env(Path::new("/home/someone"), None); let names: Vec<&str> = env.iter().map(|(n, _)| n.as_str()).collect(); assert_eq!(&names[..2], ["PATH", "HOME"]); for n in &names[2..] { assert!( - ["XDG_RUNTIME_DIR", "DBUS_SESSION_BUS_ADDRESS"].contains(n), + [ + "XDG_RUNTIME_DIR", + "DBUS_SESSION_BUS_ADDRESS", + "SSH_AUTH_SOCK" + ] + .contains(n), "{n}" ); } @@ -370,6 +396,54 @@ mod tests { assert_eq!(env[1].1, "/home/someone"); } + #[test] + fn only_an_own_ssh_agent_socket_passes() { + let dir = tempfile::tempdir().unwrap(); + let sock = dir.path().join("agent.sock"); + let _listener = std::os::unix::net::UnixListener::bind(&sock).unwrap(); + assert_eq!( + own_agent(Some(sock.as_os_str())), + Some(sock.clone().into_os_string()) + ); + // It reaches the host's environment, once. + let env = host_env(Path::new("/home/someone"), Some(sock.as_os_str())); + let agents: Vec<_> = env.iter().filter(|(n, _)| n == "SSH_AUTH_SOCK").collect(); + assert_eq!(agents.len(), 1); + assert_eq!(agents[0].1, sock.as_os_str()); + // A link to it is not it (whoever made the link chose where it + // points). + let link = dir.path().join("link.sock"); + std::os::unix::fs::symlink(&sock, &link).unwrap(); + assert_eq!(own_agent(Some(link.as_os_str())), None); + let env = host_env(Path::new("/home/someone"), Some(link.as_os_str())); + assert!(env.iter().all(|(n, _)| n != "SSH_AUTH_SOCK")); + // Named relative to here: not taken, even where it leads to one. + let here = std::env::current_dir().unwrap(); + let up = here.components().count() - 1; + let relative: std::path::PathBuf = std::iter::repeat_n("..", up) + .collect::() + .join(sock.strip_prefix("/").unwrap()); + assert!(relative.exists() && relative.is_relative(), "{relative:?}"); + assert_eq!(own_agent(Some(relative.as_os_str())), None); + // Another account's socket (the system's bus, owned by root, where + // there is one): not ours. + let system = Path::new("/run/dbus/system_bus_socket"); + if system.exists() && own_uid() != 0 { + assert_eq!(own_agent(Some(system.as_os_str())), None); + } + let file = dir.path().join("plain"); + std::fs::write(&file, "x").unwrap(); + for bad in [ + file.as_os_str(), + "agent.sock".as_ref(), + "/nonexistent/agent.sock".as_ref(), + "".as_ref(), + ] { + assert_eq!(own_agent(Some(bad)), None, "{bad:?}"); + } + assert_eq!(own_agent(None), None); + } + #[test] fn this_executable_is_a_file_on_disk() { let exe = this_executable().unwrap(); diff --git a/crates/ledger-mcp/handbook/backups.md b/crates/ledger-mcp/handbook/backups.md new file mode 100644 index 0000000..b51c09c --- /dev/null +++ b/crates/ledger-mcp/handbook/backups.md @@ -0,0 +1,9 @@ +# Backups + +- The books back themselves up: a little after each change, when they open, before they close. Each backup goes to the targets the settings name: backup.git_remote, a private repository the human owns, and backup.folder, a folder on another disk. Both is best. The backups aren't encrypted: the human's repository host and disk keep them private. +- No targets set: books.start says so. Ask the human for the repository's address and the folder, then set them with books.set_setting (who decided, and when); choosing where backups go needs books:admin, as backing up does. +- books.backup says how each target stands: when it last took the books, through which journal seq, why the last try failed, and whether its weekly restore check passed. A target that is stale, failing, or failed its check: tell the human what it says, in plain words (a disk not connected, a repository that refused). +- books.back_up_now backs up at once, for instance before the human changes computers. A backup never changes the books. +- A backup folder that took these books' backups but holds none now is a disk not connected: ask the human to connect it. Only if they say the disk is new, back up now with new_disk: true to begin the books' backup there again. +- Books restored as a copy of other books (under another name) don't back up to those books' repository (that would stop theirs): books.backup says so; ask the human for one of their own. +- Restoring is the human's to do, outside these tools: a backup comes back as new books, or replacing these (the old books are kept, never deleted), and each is proven whole before it is restored. diff --git a/crates/ledger-mcp/src/admin.rs b/crates/ledger-mcp/src/admin.rs index 6429ea0..0b9e0aa 100644 --- a/crates/ledger-mcp/src/admin.rs +++ b/crates/ledger-mcp/src/admin.rs @@ -1,6 +1,6 @@ //! The admin socket's messages (`run/admin.sock`, plan 3.1, 3.3, 3.9): how -//! the command line asks the books server to list, create and adopt books, -//! and to make, list and revoke agents' tokens. +//! the command line asks the books server to list, create, adopt and +//! restore books, and to make, list and revoke agents' tokens. //! //! ```text //! GOAL: the command line makes and adopts books without ever touching @@ -16,9 +16,10 @@ //! that says to restart. //! 3. No answer names the data root or a path in it, nor echoes a path //! the caller gave (which could be the root under another name). -//! 4. Adopting is two steps: without `confirmed` the answer is a -//! preview, and nothing is written; `confirmed: true` adopts, and -//! only books still at the head the preview read (`expect_head`). +//! 4. Adopting and restoring are two steps: without `confirmed` the +//! answer is a preview, and nothing is written; `confirmed: true` +//! adopts (restores), and only books (a backup) still at the head the +//! preview read (`expect_head`). //! 5. Nothing in a settings file is dropped silently: entries the books //! don't know are refused unless `skip_unknown`, and its notes and //! comments are in the preview. @@ -97,6 +98,24 @@ pub enum Request { #[serde(default)] skip_unknown: bool, }, + /// Restore a backup (plan 3.7): as new books, or replacing books + /// (which are moved into `legacy/`, kept). + Restore { + /// The books' id. + id: String, + /// The backup: a git repository's address, or its folder + /// (`.git` in the backup folder). + source: String, + /// Replace the books called `id`. + #[serde(default)] + replace: bool, + /// Restore (true), or only preview it. + #[serde(default)] + confirmed: bool, + /// The head the preview read: restored only if still so. + #[serde(default)] + expect_head: Option<(u64, String)>, + }, /// Make a token for an agent (plan 3.3). TokenCreate { /// The books it opens. @@ -237,6 +256,44 @@ pub struct Adopted { pub problem: Option, } +/// What a restore would bring in. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RestorePreview { + /// The backup's head, proven. + pub head: (u64, String), + /// Stored files in it, each re-hashed. + pub blobs: u64, + /// TIN vault records in it. + pub vault_files: u64, + /// Whether books of that id exist now. + pub exists: bool, + /// The id of the books it holds, as they were backed up. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub origin: Option, +} + +/// What a restore did. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Restored { + /// The books' id. + pub id: String, + /// Their head: the backup's. + pub head: (u64, String), + /// Stored files the books name, re-hashed through the store. + pub blobs_verified: u64, + /// TIN vault records restored. + pub vault_files: u64, + /// The `legacy/` folder the replaced books were moved to. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub replaced: Option, + /// The id of the books it held, as they were backed up. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub origin: Option, + /// Why the restored books didn't open, if they didn't (they are kept). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub problem: Option, +} + /// The server's answer. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case", deny_unknown_fields)] @@ -255,6 +312,10 @@ pub enum Response { Preview(AdoptPreview), /// Books adopted and open. Adopted(Adopted), + /// What a restore would bring in. + RestorePreview(RestorePreview), + /// A backup restored, and open. + Restored(Restored), /// A token made: its text, shown this once (INVARIANT 6). TokenCreated { /// The token. @@ -307,7 +368,7 @@ pub fn call_within( // A list changes nothing: its only question is whether it answered. let look = match request { Request::List | Request::TokenList => None, - Request::Create { .. } | Request::Adopt { .. } => { + Request::Create { .. } | Request::Adopt { .. } | Request::Restore { .. } => { Some("`wanderbooks books list` shows the books there are") } Request::TokenCreate { .. } | Request::TokenRevoke { .. } => { @@ -417,6 +478,24 @@ mod tests { } } + #[test] + fn a_restore_request_is_a_preview_unless_confirmed() { + let r: Envelope = serde_json::from_value( + json!({"proto": 1, "op": "restore", "id": "acme", "source": "/backups/acme.git"}), + ) + .unwrap(); + assert_eq!( + r.request, + Request::Restore { + id: "acme".into(), + source: "/backups/acme.git".into(), + replace: false, + confirmed: false, + expect_head: None, + } + ); + } + #[test] fn unknown_operations_and_fields_are_refused() { for bad in [ @@ -455,6 +534,22 @@ mod tests { text: TokenText("wb_00".into()), }, Response::Tokens(vec![]), + Response::RestorePreview(RestorePreview { + head: (3, "ab".into()), + blobs: 2, + vault_files: 1, + exists: true, + origin: Some("acme".into()), + }), + Response::Restored(Restored { + id: "acme".into(), + head: (3, "ab".into()), + blobs_verified: 2, + vault_files: 1, + replaced: Some("acme-20261019-120000".into()), + origin: Some("acme".into()), + problem: None, + }), ] { let text = serde_json::to_string(&r).unwrap(); assert_eq!(serde_json::from_str::(&text).unwrap(), r); diff --git a/crates/ledger-mcp/src/backup.rs b/crates/ledger-mcp/src/backup.rs new file mode 100644 index 0000000..3f2080e --- /dev/null +++ b/crates/ledger-mcp/src/backup.rs @@ -0,0 +1,994 @@ +//! The server's backups of a book (plan 3.7, D-30, D-31): made a while +//! after the books are written (when the writing stops), when they open, +//! before they close, and when asked; each pushed to the targets the +//! book's settings name; each recorded in the book's `ops/backup.jsonl`; +//! each target's backup drilled weekly. +//! +//! ```text +//! GOAL: every write reaches the owner's backups soon after it is made, +//! and the owner (and agent) can see that it did, or why not. +//! SOURCE: plan 3.7 (triggers, "ops/backup.jsonl", books.backup, Needs +//! attention for a target more than 24 h behind, drills), D-30 (a +//! git repository and a folder), D-31 (plain git). +//! INVARIANTS: +//! 1. A backup is committed under the store's lock (ledger_store::backup +//! INVARIANT 1; its stored files staged before, outside it) and +//! pushed outside it: the books are never held while the network is +//! slow. One run or drill at a time. +//! 2. Every commit, push and drill is a record of `ops/backup.jsonl` (a +//! chain log, not the journal: a backup never moves the books' head), +//! each target's outcome a record of its own, keyed by the target +//! itself (a new folder or address starts with no standing). +//! 3. A target is stale when it lacks the books' head and the first try +//! to back up what it lacks (the first commit or failure since its +//! last good push, or since its first try if it never had one, or +//! since the failed commits that kept it from any) is more than +//! STALE_AFTER old; its last error is its last push's, or a +//! later failed commit's. +//! 4. Nothing here names a path of the data root to an agent. +//! 5. Every target that holds a backup is drilled (ledger_store::backup:: +//! drill: cloned, proven whole, its journal the live one's beginning) +//! once DRILL_EVERY has passed since its last good drill, or +//! RETRY_AFTER since a failed one (at the server's next look for +//! drills); a failed drill is said in the standing until one passes. +//! 6. A target whose push failed lately (RETRY_AFTER) is left out of a +//! backup after a write, and only that one; a folder target's +//! repository is made only while the target never took a backup of +//! these books (one gone since is a disk not connected), restored +//! books counting the backups of the books they were. +//! 7. Books restored beside the books they were (under another id) never +//! push to the repository those books back up to (it would stop +//! theirs): it is paused for them, and said, until another is named. +//! 8. A damaged backup record never keeps the books shut: it is set +//! aside and a new one begun. +//! ``` + +use crate::server::SharedStore; +use ledger_store::backup::{self, BackupError, Git, Target}; +use ledger_store::chain::ChainLog; +use serde::Serialize; +use serde_json::{Map, Value, json}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; + +/// The book's backup record, in its `ops/`. +pub const LOG_FILE: &str = "backup.jsonl"; +/// The record's genesis text. +const GENESIS: &str = "wanderbooks-backup-v1"; +/// How long a target may lack the books' head before it is stale. +pub const STALE_AFTER: chrono::Duration = chrono::Duration::hours(24); +/// How often each target's backup is drilled (plan 3.7: weekly). +pub const DRILL_EVERY: chrono::Duration = chrono::Duration::days(7); +/// How long after a failed run (or drill) the next is tried. +pub const RETRY_AFTER: std::time::Duration = std::time::Duration::from_secs(15 * 60); +/// The most of an error kept. +const ERROR_MAX: usize = 500; + +/// A book's backups. +pub struct Backups { + book: String, + dir: PathBuf, + store: Arc, + log: Mutex, + /// One run at a time. + running: tokio::sync::Mutex<()>, +} + +/// One target's standing (INVARIANT 3). +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct TargetStatus { + /// "repository" or "folder". + pub target: String, + /// Where (the repository's address, or "the backup folder"). + pub at: String, + /// When it last took a push, if ever (UTC). + pub last_ok_at: Option, + /// The journal seq it holds, if any. + pub through_seq: Option, + /// Whether it is stale (INVARIANT 3). + pub stale: bool, + /// Why its last backup failed, if it did. + pub last_error: Option, + /// When its backup was last drilled (UTC), if ever. + pub last_drill_at: Option, + /// Why its last drill failed, if it did. + pub drill_error: Option, + /// Paused for books restored as a copy of others (INVARIANT 7). + pub paused: bool, +} + +/// The books' backup standing. +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct Status { + /// The books' head now. + pub head_seq: u64, + /// Each target the settings name. + pub targets: Vec, + /// None set up: what to do. + pub note: Option, +} + +/// A target the settings name. +#[derive(Debug, Clone)] +struct Named { + target: Target, + /// Where, as an agent is told it. + at: String, + /// The target itself, as its records name it (INVARIANT 2): a digest + /// of where it is (never a path an agent is shown). + key: String, +} + +/// The targets the book's settings name (D-30). +fn targets(store: &ledger_store::Store, book: &str) -> Vec { + use sha2::{Digest, Sha256}; + let s = store.ledger().settings(); + let mut out = Vec::new(); + if let Some(url) = s.backup_git_remote() { + out.push((Target::Remote(url.to_string()), url.to_string())); + } + if let Some(folder) = s.backup_folder() { + out.push(( + Target::Folder(PathBuf::from(folder)), + "the backup folder".to_string(), + )); + } + out.into_iter() + .map(|(target, at)| Named { + key: Sha256::digest(target.source(book).as_bytes())[..8] + .iter() + .map(|b| format!("{b:02x}")) + .collect(), + target, + at, + }) + .collect() +} + +impl Backups { + /// The backups of the book `book`, kept in `dir`, served from `store`. + pub fn open(book: &str, dir: &Path, store: Arc) -> Result { + ledger_store::ops::prepare(dir) + .map_err(|e| format!("its backup record can't be kept: {e}"))?; + let path = ledger_store::ops::ops_dir(dir).join(LOG_FILE); + let (log, carried) = match ChainLog::open(&path, GENESIS) { + Ok((log, _)) => (log, Vec::new()), + // INVARIANT 8: set aside, begun again, what it knew of whose + // backups the targets hold carried into the new one. + Err(ledger_store::chain::ChainError::Damaged { .. }) => { + let known = known_before_damage(&path); + let aside = path.with_extension(format!( + "jsonl.damaged-{}", + chrono::Utc::now().format("%Y%m%d-%H%M%S") + )); + eprintln!( + "wanderbooks server: a backup record was damaged: set aside, begun again" + ); + std::fs::rename(&path, &aside) + .map_err(|e| format!("its backup record can't be set aside: {}", e.kind()))?; + let log = ChainLog::open(&path, GENESIS) + .map_err(|e| format!("its backup record can't be used: {e}"))? + .0; + (log, known) + } + Err(e) => return Err(format!("its backup record can't be used: {e}")), + }; + let me = Self { + book: book.to_string(), + dir: dir.to_path_buf(), + store, + log: Mutex::new(log), + running: tokio::sync::Mutex::new(()), + }; + for fields in carried { + me.record(fields); + } + // Books just restored: whose they were (INVARIANTS 6, 7). + let marker = ledger_store::ops::ops_dir(dir).join(ledger_store::root::RESTORED_FROM); + if let Ok(origin) = std::fs::read_to_string(&marker) { + let origin = Some(origin.trim()).filter(|o| !o.is_empty()); + let named = me + .store + .try_with_store(|s| targets(s, book)) + .ok_or("its backups can't be read yet: the books are busy")?; + me.restored(origin, &named); + std::fs::remove_file(&marker) + .map_err(|e| format!("its backup record can't be kept: {}", e.kind()))?; + } + Ok(me) + } + + fn record(&self, fields: Value) { + let Value::Object(mut map) = fields else { + return; + }; + // INVARIANT 4: a git or store message may quote a path in the root. + if let Some(Value::String(e)) = map.get_mut("error") { + *e = hide(e, &self.dir); + } + let mut log = self.log.lock().unwrap_or_else(|e| e.into_inner()); + if let Err(e) = log.append(map) { + eprintln!("wanderbooks server: the books' backup record couldn't be written: {e}"); + } + } + + fn records(&self) -> Vec> { + ledger_store::chain::read( + &ledger_store::ops::ops_dir(&self.dir).join(LOG_FILE), + GENESIS, + ) + .map(|r| r.records) + .unwrap_or_default() + } + + /// The targets the settings name now. + async fn named(&self) -> Vec { + let (store, book) = (self.store.clone(), self.book.clone()); + tokio::task::spawn_blocking(move || store.with_store(|s| targets(s, &book)).ok()) + .await + .ok() + .flatten() + .unwrap_or_default() + } + + /// Note that these books were just restored from a backup of the books + /// `origin` (INVARIANTS 6, 7), `named` the targets they name: the same + /// books (their targets took their backups), or others' (the + /// repository those back up to is paused for these). + fn restored(&self, origin: Option<&str>, named: &[Named]) { + let same = origin == Some(self.book.as_str()); + let keys = |remote: bool| -> Vec { + named + .iter() + .filter(|n| !remote || matches!(n.target, Target::Remote(_))) + .map(|n| n.key.clone()) + .collect() + }; + if same { + self.record(json!({"kind": "restored", "origin": origin, "held": keys(false)})); + } else { + self.record(json!({"kind": "restored", "origin": origin, "paused": keys(true)})); + } + } + + /// Wait for any run or drill to end, and start none while the guard + /// lives: books being replaced by a restore push nothing more. + pub async fn pause(&self) -> tokio::sync::MutexGuard<'_, ()> { + self.running.lock().await + } + + /// The books' head now. + pub async fn head(&self) -> Option<(u64, String)> { + let store = self.store.clone(); + tokio::task::spawn_blocking(move || store.with_store(|s| s.head()).ok()) + .await + .ok() + .flatten() + } + + /// Back the books up now (INVARIANTS 1, 2): `why` is recorded (a + /// write, open, close, asked). The status after. + pub async fn run(&self, why: &str) -> Result { + self.run_with(why, false).await + } + + /// [`Self::run`]; with `new_disk`, a folder target that took these + /// books' backups and holds none now is begun again there (the owner + /// says it is a new disk). + pub async fn run_with(&self, why: &str, new_disk: bool) -> Result { + let _one = self.running.lock().await; + if self.named().await.is_empty() { + return Ok(self.status().await); + } + let (store, dir, book) = (self.store.clone(), self.dir.clone(), self.book.clone()); + let made = tokio::task::spawn_blocking(move || { + let git = Git::find()?; + // The stored files first, outside the lock: they never change. + // Only a head start (a file being written can trip it): the + // commit stages them all again. + let _ = backup::stage_blobs(&git, &dir); + // The commit, under it. + store + .with_store(|s| -> Result<_, BackupError> { + let made = backup::commit(&git, &dir, &book, s.head())?; + Ok((made, targets(s, &book), s.head())) + }) + .map_err(BackupError::Damaged) + .and_then(|r| r) + }) + .await + .unwrap_or(Err(BackupError::Damaged("the backup stopped".into()))); + let (made, named, head) = match made { + Ok(m) => m, + Err(e) => { + let e = hide(&e.to_string(), &self.dir); + self.record(json!({"kind": "commit", "why": why, "ok": false, "error": e})); + return Err(e); + } + }; + if let Some(m) = &made { + self.record( + json!({"kind": "commit", "why": why, "ok": true, "commit": m.commit, + "head": {"seq": m.head.0, "hash": m.head.1}}), + ); + } + // Each target, outside the lock; after a write, one that failed + // lately waits out RETRY_AFTER (the others don't wait for it). + let records = self.records(); + let now = chrono::Utc::now(); + for n in named { + if paused(&n.key, &records) || (why == "write" && failed_lately(&n.key, &records, now)) + { + continue; + } + // Its repository is made only where it never took a backup. + let may_create = new_disk || !held_before(&n.key, &records); + let (dir, book, t) = (self.dir.clone(), self.book.clone(), n.target.clone()); + let pushed = tokio::task::spawn_blocking(move || { + let git = Git::find()?; + backup::push(&git, &dir, &book, &t, may_create) + }) + .await + .unwrap_or(Err(BackupError::Damaged("the push stopped".into()))); + match pushed { + Ok(commit) => self.record( + json!({"kind": "push", "why": why, "target": n.target.kind(), + "at": n.at, "key": n.key, "ok": true, "commit": commit, + "head": {"seq": head.0, "hash": head.1}}), + ), + Err(e) => self.record( + json!({"kind": "push", "why": why, "target": n.target.kind(), + "at": n.at, "key": n.key, "ok": false, "error": e.to_string()}), + ), + } + } + Ok(self.status().await) + } + + /// Whether a drill is due (INVARIANT 5). + pub async fn drill_due(&self) -> bool { + let records = self.records(); + let now = chrono::Utc::now(); + self.named() + .await + .iter() + .any(|n| drill_due(&n.key, &records, now)) + } + + /// Drill each target whose drill is due (INVARIANT 5), each recorded. + pub async fn drill(&self) { + let _one = self.running.lock().await; + let records = self.records(); + let now = chrono::Utc::now(); + for n in self.named().await { + if paused(&n.key, &records) || !drill_due(&n.key, &records, now) { + continue; + } + let (dir, book, t) = (self.dir.clone(), self.book.clone(), n.target.clone()); + let drilled = tokio::task::spawn_blocking(move || { + let scratch = ledger_store::ops::ops_dir(&dir).join(format!("drill-{}", t.kind())); + let _ = std::fs::remove_dir_all(&scratch); + let git = Git::find()?; + backup::drill( + &git, + &t, + &book, + &dir.join(ledger_store::JOURNAL_FILE), + &scratch, + ) + }) + .await + .unwrap_or(Err(BackupError::Damaged("the drill stopped".into()))); + match drilled { + Ok(head) => { + self.record(json!({"kind": "drill", "target": n.target.kind(), + "at": n.at, "key": n.key, "ok": true, "head": {"seq": head.0, "hash": head.1}})) + } + Err(e) => self.record(json!({"kind": "drill", "target": n.target.kind(), + "at": n.at, "key": n.key, "ok": false, "error": e.to_string()})), + } + } + } + + /// The books' backup standing (INVARIANT 3). + pub async fn status(&self) -> Status { + let store = self.store.clone(); + let (book, dir) = (self.book.clone(), self.dir.clone()); + tokio::task::spawn_blocking(move || store.with_store(|s| standing(s, &book, &dir)).ok()) + .await + .ok() + .flatten() + .unwrap_or_else(|| standing_of(&[], &[], 0, chrono::Utc::now())) + } + + /// The standing, read from `store` (held by the caller: books.start). + pub fn standing(&self, store: &ledger_store::Store) -> Status { + standing(store, &self.book, &self.dir) + } +} + +fn standing(store: &ledger_store::Store, book: &str, dir: &Path) -> Status { + let records = + ledger_store::chain::read(&ledger_store::ops::ops_dir(dir).join(LOG_FILE), GENESIS) + .map(|r| r.records) + .unwrap_or_default(); + standing_of( + &targets(store, book), + &records, + store.head().0, + chrono::Utc::now(), + ) +} + +fn standing_of( + named: &[Named], + records: &[Map], + head_seq: u64, + now: chrono::DateTime, +) -> Status { + Status { + head_seq, + note: named.is_empty().then(|| { + "no backups are set up: set backup.git_remote (a private repository) and backup.folder (a folder on another disk) with books.set_setting".to_string() + }), + targets: named + .iter() + .map(|n| status_of(n.target.kind(), &n.at, &n.key, records, head_seq, now)) + .collect(), + } +} + +impl Status { + /// Whether the human should hear of it: none set up, or one stale, + /// failing, or failing its drill. + pub fn needs_attention(&self) -> bool { + self.note.is_some() + || self + .targets + .iter() + .any(|t| t.stale || t.last_error.is_some() || t.drill_error.is_some()) + } +} + +/// `text` with the book's folder `dir`, and the data root above it, named +/// in place of their paths (INVARIANT 4); at most ERROR_MAX characters. +fn hide(text: &str, dir: &Path) -> String { + let mut text = text.to_string(); + let real = std::fs::canonicalize(dir).unwrap_or_else(|_| dir.to_path_buf()); + for d in [dir, real.as_path()] { + text = text.replace(&d.display().to_string(), "the book's folder"); + if let Some(root) = d.parent().and_then(Path::parent) { + text = text.replace(&root.display().to_string(), ledger_store::root::THE_ROOT); + } + } + text.chars().take(ERROR_MAX).collect() +} + +fn is(r: &Map, kind: &str) -> bool { + r.get("kind").and_then(Value::as_str) == Some(kind) +} + +fn ok(r: &Map) -> bool { + r.get("ok") == Some(&Value::Bool(true)) +} + +fn of(r: &Map, key: &str) -> bool { + r.get("key").and_then(Value::as_str) == Some(key) +} + +fn ts(r: &Map) -> Option> { + r.get("ts") + .and_then(Value::as_str) + .and_then(|t| chrono::DateTime::parse_from_rfc3339(t).ok()) + .map(|t| t.with_timezone(&chrono::Utc)) +} + +fn text(r: &Map, field: &str) -> Option { + r.get(field).and_then(Value::as_str).map(str::to_string) +} + +/// What a damaged record knew, up to its first line that can't be read at +/// all (a line read but out of the chain still counts: what it adds only +/// holds more back), of whose backups the +/// targets hold: its restored notes, and each target that took a backup +/// (as held). +fn known_before_damage(path: &Path) -> Vec { + let text = std::fs::read(path).unwrap_or_default(); + let mut held: Vec = Vec::new(); + let mut out = Vec::new(); + for line in text.split(|b| *b == b'\n') { + let Ok(Value::Object(r)) = serde_json::from_slice::(line) else { + break; + }; + if is(&r, "push") && ok(&r) { + if let Some(k) = text_field(&r, "key") + && !held.contains(&k) + { + held.push(k); + } + } else if is(&r, "restored") { + let mut r = r.clone(); + for k in ["seq", "ts", "prev", "hash"] { + r.remove(k); + } + out.push(Value::Object(r)); + } + } + if !held.is_empty() { + out.push(json!({"kind": "restored", "carried": true, "held": held})); + } + out +} + +fn text_field(r: &Map, field: &str) -> Option { + r.get(field).and_then(Value::as_str).map(str::to_string) +} + +/// Whether the target `key` is paused for books restored beside others +/// (INVARIANT 7). +fn paused(key: &str, records: &[Map]) -> bool { + records.iter().any(|r| { + is(r, "restored") + && r.get("paused") + .and_then(Value::as_array) + .is_some_and(|keys| keys.iter().any(|k| k.as_str() == Some(key))) + }) +} + +/// Whether the target `key` took a backup of these books (INVARIANT 6). +fn held_before(key: &str, records: &[Map]) -> bool { + records.iter().any(|r| { + (is(r, "push") && of(r, key) && ok(r)) + || (is(r, "restored") + && r.get("held") + .and_then(Value::as_array) + .is_some_and(|keys| keys.iter().any(|k| k.as_str() == Some(key)))) + }) +} + +/// What books restored beside others are told of the repository paused +/// for them. +pub const PAUSED: &str = "not backed up here: these books were restored as a copy of the books this repository keeps, and backing up here would stop theirs; name a repository of their own (backup.git_remote)"; + +/// Whether the target `key`'s last push failed less than RETRY_AFTER ago. +fn failed_lately( + key: &str, + records: &[Map], + now: chrono::DateTime, +) -> bool { + records + .iter() + .rfind(|r| is(r, "push") && of(r, key)) + .filter(|r| !ok(r)) + .and_then(ts) + .is_some_and(|at| { + now.signed_duration_since(at) + < chrono::Duration::from_std(RETRY_AFTER).unwrap_or(chrono::Duration::zero()) + }) +} + +/// Whether the target `key`'s drill is due (INVARIANT 5). +fn drill_due( + key: &str, + records: &[Map], + now: chrono::DateTime, +) -> bool { + if !records.iter().any(|r| is(r, "push") && of(r, key) && ok(r)) { + return false; + } + match records.iter().rfind(|r| is(r, "drill") && of(r, key)) { + None => true, + Some(d) => ts(d).is_none_or(|at| { + let wait = if ok(d) { + DRILL_EVERY + } else { + chrono::Duration::from_std(RETRY_AFTER).unwrap_or(DRILL_EVERY) + }; + now.signed_duration_since(at) >= wait + }), + } +} + +/// A target's standing from the backup record (INVARIANT 3). +fn status_of( + kind: &str, + at: &str, + key: &str, + records: &[Map], + head_seq: u64, + now: chrono::DateTime, +) -> TargetStatus { + let last_ok = records + .iter() + .rposition(|r| is(r, "push") && of(r, key) && ok(r)); + let through_seq = last_ok + .and_then(|i| records[i].get("head")) + .and_then(|h| h.get("seq")) + .and_then(Value::as_u64); + // Since its last good push (or, never having had one, since its first + // try; or, never tried, the failed commits that kept it from one): + // this target's tries, and the commits. + let from = match last_ok { + Some(i) => i + 1, + None => records.iter().position(|r| of(r, key)).unwrap_or_else(|| { + records + .iter() + .rposition(|r| !(is(r, "commit") && !ok(r))) + .map_or(0, |i| i + 1) + }), + }; + let since: Vec<&Map> = records[from..] + .iter() + .filter(|r| is(r, "commit") || (is(r, "push") && of(r, key))) + .collect(); + let behind = through_seq.is_none_or(|s| s < head_seq); + let old = since + .first() + .and_then(|r| ts(r)) + .is_some_and(|t| now.signed_duration_since(t) > STALE_AFTER); + let last_error = since + .iter() + .rev() + .find(|r| !ok(r) || is(r, "push")) + .filter(|r| !ok(r)) + .and_then(|r| text(r, "error")); + let drill = records.iter().rfind(|r| is(r, "drill") && of(r, key)); + let paused = paused(key, records); + TargetStatus { + target: kind.to_string(), + at: at.to_string(), + last_ok_at: last_ok.and_then(|i| text(&records[i], "ts")), + through_seq, + stale: behind && old && head_seq > 0 && !paused, + last_error: if paused { + Some(PAUSED.to_string()) + } else { + last_error + }, + last_drill_at: drill.and_then(|r| text(r, "ts")), + drill_error: drill.filter(|r| !ok(r)).and_then(|r| text(r, "error")), + paused, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn no_error_names_the_root() { + let dir = Path::new("/home/u/.local/share/wanderbooks/books/acme"); + assert_eq!( + hide( + "fatal: '/home/u/.local/share/wanderbooks/books/acme/archive.git' and /home/u/.local/share/wanderbooks/run", + dir + ), + format!( + "fatal: 'the book's folder/archive.git' and {}/run", + ledger_store::root::THE_ROOT + ) + ); + assert_eq!(hide(&"x".repeat(2000), dir).len(), ERROR_MAX); + } + + /// A record of `kind` for target `key`, `ago` before `now`. + fn rec( + now: chrono::DateTime, + kind: &str, + key: &str, + ok: bool, + seq: u64, + ago: chrono::Duration, + ) -> Map { + let mut m = Map::new(); + m.insert("kind".into(), json!(kind)); + if kind != "commit" { + m.insert("key".into(), json!(key)); + } + m.insert("ok".into(), json!(ok)); + m.insert("ts".into(), json!((now - ago).to_rfc3339())); + if ok { + m.insert("head".into(), json!({"seq": seq, "hash": "h"})); + } else { + m.insert("error".into(), json!(format!("{kind} failed"))); + } + m + } + + #[test] + fn a_target_is_stale_only_when_behind_since_a_day() { + let now = chrono::Utc::now(); + let r = |kind, ok, seq, ago| rec(now, kind, "k", ok, seq, ago); + let st = |records: &[Map], head| { + status_of("folder", "f", "k", records, head, now) + }; + let day = chrono::Duration::hours(24); + let hour = chrono::Duration::hours(1); + // Nothing tried yet: not stale. + assert!(!st(&[], 0).stale); + assert!(!st(&[], 4).stale); + // Never pushed, the first try a day ago: stale; an hour ago: not. + assert!( + st( + &[ + r("commit", true, 3, day + hour), + r("push", false, 0, day + hour) + ], + 3 + ) + .stale + ); + assert!(!st(&[r("commit", true, 3, hour), r("push", false, 0, hour)], 3).stale); + // Pushed long ago, and quiet since: a new write isn't stale at once. + let quiet = [r("commit", true, 3, day * 3), r("push", true, 3, day * 3)]; + assert!(!st(&quiet, 4).stale); + // ... nor while it holds the head. + assert!(!st(&quiet, 3).stale); + // Behind since a commit a day old, its pushes failing: stale. + let failing = [ + r("push", true, 3, day * 3), + r("commit", true, 4, day + hour), + r("push", false, 0, day), + r("push", false, 0, hour), + ]; + let t = st(&failing, 4); + assert!(t.stale); + assert_eq!( + (t.through_seq, t.last_error.as_deref()), + (Some(3), Some("push failed")) + ); + // A good push ends the error. + let mended = [ + &failing[..], + &[r("push", true, 4, chrono::Duration::zero())], + ] + .concat(); + let t = st(&mended, 4); + assert_eq!( + (t.stale, t.last_error, t.through_seq), + (false, None, Some(4)) + ); + // A failed commit is every target's error, until a good push. + let t = st(&[r("push", true, 3, day), r("commit", false, 0, hour)], 4); + assert_eq!(t.last_error.as_deref(), Some("commit failed")); + // A good commit after a failed push doesn't hide the failure. + let t = st(&[r("push", false, 0, hour), r("commit", true, 4, hour)], 4); + assert_eq!(t.last_error.as_deref(), Some("push failed")); + // Another target's records are not its: a new folder starts clean. + let other = [ + rec(now, "push", "other", true, 3, hour), + rec(now, "push", "other", false, 0, hour), + ]; + let t = st(&other, 3); + assert_eq!( + (t.through_seq, t.last_error, t.last_ok_at), + (None, None, None) + ); + } + + #[test] + fn a_target_failing_lately_is_left_out_of_a_write_alone() { + let now = chrono::Utc::now(); + let retry = chrono::Duration::from_std(RETRY_AFTER).unwrap(); + let second = chrono::Duration::seconds(1); + let failed = |ago| rec(now, "push", "k", false, 0, ago); + assert!(failed_lately("k", &[failed(retry - second)], now)); + assert!(!failed_lately("k", &[failed(retry)], now)); + assert!(!failed_lately( + "k", + &[failed(second), rec(now, "push", "k", true, 3, second)], + now + )); + assert!(!failed_lately("other", &[failed(second)], now)); + assert!(!failed_lately("k", &[], now)); + } + + #[test] + fn commits_failing_from_the_start_are_every_targets_trouble() { + let now = chrono::Utc::now(); + let day = chrono::Duration::hours(24); + let hour = chrono::Duration::hours(1); + // git missing (say) from the first run on: never a push of its own. + let failing = [ + rec(now, "commit", "", false, 0, day + hour), + rec(now, "commit", "", false, 0, hour), + ]; + let t = status_of("folder", "f", "k", &failing, 3, now); + assert_eq!(t.last_error.as_deref(), Some("commit failed")); + assert!(t.stale, "{t:?}"); + // Failing an hour only: said, not yet stale. + let t = status_of("folder", "f", "k", &failing[1..], 3, now); + assert_eq!( + (t.last_error.as_deref(), t.stale), + (Some("commit failed"), false) + ); + // A commit that worked since (its push still to come): no error. + let mended = [ + &failing[..], + &[rec(now, "commit", "", true, 3, chrono::Duration::zero())], + ] + .concat(); + assert_eq!( + status_of("folder", "f", "k", &mended, 3, now).last_error, + None + ); + } + + #[test] + fn restored_books_know_which_backups_were_theirs() { + let now = chrono::Utc::now(); + let restored = |field: &str, keys: &[&str]| { + let mut m = Map::new(); + m.insert("kind".into(), json!("restored")); + m.insert(field.into(), json!(keys)); + m.insert("ts".into(), json!(now.to_rfc3339())); + m + }; + // The same books: their targets took their backups. + let same = [restored("held", &["k"])]; + assert!(held_before("k", &same)); + assert!(!held_before("new", &same)); + assert!(!paused("k", &same)); + // Others' (restored beside them): their repository is paused, said. + let beside = [restored("paused", &["r"])]; + assert!(paused("r", &beside)); + assert!(!paused("k", &beside)); + assert!(!held_before("r", &beside)); + let t = status_of("repository", "x", "r", &beside, 9, now); + assert_eq!(t.last_error.as_deref(), Some(PAUSED)); + assert!(!t.stale && t.paused); + // A push of its own counts as held, too. + assert!(held_before( + "k", + &[rec(now, "push", "k", true, 3, chrono::Duration::zero())] + )); + assert!(!held_before( + "k", + &[rec(now, "push", "k", false, 0, chrono::Duration::zero())] + )); + } + + #[test] + fn a_target_named_later_starts_its_own_clock() { + let now = chrono::Utc::now(); + let day = chrono::Duration::hours(24); + let hour = chrono::Duration::hours(1); + // Commits for a week, to another target; then this one's first try. + let records = [ + rec(now, "commit", "", true, 2, day * 7), + rec(now, "push", "old", true, 2, day * 7), + rec(now, "commit", "", true, 3, day * 2), + rec(now, "push", "k", false, 0, hour), + ]; + let t = status_of("folder", "f", "k", &records, 3, now); + assert!(!t.stale, "{t:?}"); + assert_eq!(t.last_error.as_deref(), Some("push failed")); + // Holding the head: never stale; one behind since long: stale. + let held = [ + rec(now, "push", "k", true, 3, day * 3), + rec(now, "push", "k", false, 0, day * 2), + ]; + assert!(!status_of("folder", "f", "k", &held, 3, now).stale); + assert!(status_of("folder", "f", "k", &held, 4, now).stale); + } + + #[test] + fn drills_are_due_weekly_after_a_pass_and_soon_after_a_failure() { + let now = chrono::Utc::now(); + let r = |kind, ok, ago| rec(now, kind, "k", ok, 3, ago); + let hour = chrono::Duration::hours(1); + let week = DRILL_EVERY; + let retry = chrono::Duration::from_std(RETRY_AFTER).unwrap(); + // Nothing backed up there: nothing to drill. + assert!(!drill_due("k", &[], now)); + assert!(!drill_due("k", &[r("push", false, hour)], now)); + // Backed up, never drilled: due. + let pushed = r("push", true, hour); + assert!(drill_due("k", std::slice::from_ref(&pushed), now)); + // Passed: due a week after. + assert!(!drill_due( + "k", + &[pushed.clone(), r("drill", true, week - hour)], + now + )); + assert!(drill_due( + "k", + &[pushed.clone(), r("drill", true, week)], + now + )); + // Failed: tried again RETRY_AFTER after. + let t = drill_due( + "k", + &[ + pushed.clone(), + r("drill", false, retry - chrono::Duration::seconds(1)), + ], + now, + ); + assert!(!t); + assert!(drill_due( + "k", + &[pushed.clone(), r("drill", false, retry)], + now + )); + // Another target's drill isn't its, nor its push. + assert!(drill_due( + "k", + &[pushed.clone(), rec(now, "drill", "o", true, 3, hour)], + now + )); + assert!(!drill_due( + "k", + &[rec(now, "push", "o", true, 3, hour)], + now + )); + // A failed drill is said until one passes. + let t = status_of( + "folder", + "f", + "k", + &[pushed.clone(), r("drill", false, hour)], + 3, + now, + ); + assert_eq!(t.drill_error.as_deref(), Some("drill failed")); + assert!(t.last_drill_at.is_some()); + let t = status_of( + "folder", + "f", + "k", + &[pushed, r("drill", false, hour), r("drill", true, hour)], + 3, + now, + ); + assert_eq!(t.drill_error, None); + } + + #[test] + fn each_kind_of_trouble_needs_attention_alone() { + let clean = TargetStatus { + target: "folder".into(), + at: "f".into(), + last_ok_at: Some("t".into()), + through_seq: Some(3), + stale: false, + last_error: None, + last_drill_at: None, + drill_error: None, + paused: false, + }; + let status = |t: TargetStatus| Status { + head_seq: 3, + targets: vec![t], + note: None, + }; + assert!(!status(clean.clone()).needs_attention()); + assert!( + status(TargetStatus { + stale: true, + ..clean.clone() + }) + .needs_attention() + ); + assert!( + status(TargetStatus { + last_error: Some("e".into()), + ..clean.clone() + }) + .needs_attention() + ); + assert!( + status(TargetStatus { + drill_error: Some("e".into()), + ..clean.clone() + }) + .needs_attention() + ); + let none = Status { + head_seq: 3, + targets: vec![], + note: Some("n".into()), + }; + assert!(none.needs_attention()); + } +} diff --git a/crates/ledger-mcp/src/handbook.rs b/crates/ledger-mcp/src/handbook.rs index 5938283..ad50d0e 100644 --- a/crates/ledger-mcp/src/handbook.rs +++ b/crates/ledger-mcp/src/handbook.rs @@ -81,6 +81,7 @@ pub const SECTIONS: &[(&str, &str)] = &[ include_str!("../handbook/settings-and-conventions.md"), ), ("viewer", include_str!("../handbook/viewer.md")), + ("backups", include_str!("../handbook/backups.md")), ( "rules-that-change", include_str!("../handbook/rules-that-change.md"), diff --git a/crates/ledger-mcp/src/lib.rs b/crates/ledger-mcp/src/lib.rs index e2602ba..b5248a6 100644 --- a/crates/ledger-mcp/src/lib.rs +++ b/crates/ledger-mcp/src/lib.rs @@ -16,6 +16,7 @@ pub mod access; pub mod admin; +pub mod backup; mod handbook; #[cfg(unix)] pub mod local; diff --git a/crates/ledger-mcp/src/local.rs b/crates/ledger-mcp/src/local.rs index 0a86855..2efb578 100644 --- a/crates/ledger-mcp/src/local.rs +++ b/crates/ledger-mcp/src/local.rs @@ -75,7 +75,7 @@ use std::os::fd::AsRawFd; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, RwLock}; -use std::time::Duration; +use std::time::{Duration, Instant}; use tokio_util::sync::CancellationToken; use tokio_util::task::TaskTracker; @@ -103,6 +103,18 @@ pub const REFUSED_HEADER: &str = "x-wanderbooks-refused"; /// How often each book's lapsed uploads are swept. const SWEEP_EVERY: Duration = Duration::from_secs(10); +/// How long the books stay unwritten before they are backed up. +const BACKUP_QUIET: Duration = Duration::from_secs(30); +/// How often the books are looked at for a backup due. +const BACKUP_LOOK: Duration = Duration::from_secs(2); +/// How often each book is looked at for a drill due. +const DRILL_LOOK: Duration = Duration::from_secs(3600); +/// How long after books are first seen open their first drill is looked +/// for (a drill that fell due while they were closed): long enough that a +/// backup on opening comes first. +const DRILL_FIRST: Duration = Duration::from_secs(60); +/// The most the last backup at a close may take. +const BACKUP_AT_CLOSE: Duration = Duration::from_secs(60); type Service = StreamableHttpService; @@ -122,6 +134,8 @@ enum Book { uploads: ledger_store::uploads::Uploads, /// The files it made for agents (`/b/{id}/x/{export}`). exports: ledger_store::exports::Exports, + /// Its backups (backup.rs). + backups: Arc, }, /// Not opened, with why (INVARIANT 3). Failed(String), @@ -160,6 +174,14 @@ struct Books { /// Which token opened each MCP session: a session is used only with /// the token that opened it (INVARIANT 8). session_tokens: Mutex>, + /// How long the books stay unwritten before a backup is made + /// (`--backup-quiet`, for tests; else BACKUP_QUIET). + backup_quiet: Duration, + /// How long after books are first seen open a drill is first looked + /// for (`--drill-first`, for tests; else DRILL_FIRST). + drill_first: Duration, + /// Held through a restore: one at a time. + restoring: Mutex<()>, } impl Books { @@ -284,7 +306,17 @@ impl Books { // INVARIANT 2): a viewer can show its presence from the start. ledger_store::ops::prepare(store.data_dir()).map_err(|e| self.hide(e.to_string()))?; let live = Arc::new(Mutex::new(live)); + let book_dir = store.data_dir().to_path_buf(); + let book_id = book_dir + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_default(); let store = SharedStore::new(store); + let backups = Arc::new( + crate::backup::Backups::open(&book_id, &book_dir, store.clone()) + .map_err(|e| self.hide(e))?, + ); + let server_backups = backups.clone(); let log = access.clone(); let shared = store.clone(); let config = StreamableHttpServerConfig::default() @@ -301,7 +333,8 @@ impl Books { .with_layout(layout.clone()) .with_access(log.clone()) .with_uploads(server_uploads.clone()) - .with_exports(server_exports.clone()), + .with_exports(server_exports.clone()) + .with_backups(server_backups.clone()), ) }, Arc::new(sessions()), @@ -313,6 +346,7 @@ impl Books { access, uploads, exports, + backups, }) } @@ -449,6 +483,13 @@ impl Books { Err(e) => refused(e), } } + Request::Restore { + id, + source, + replace, + confirmed, + expect_head, + } => self.restore(id, source, replace, confirmed, expect_head), Request::Adopt { id, source, @@ -589,6 +630,119 @@ impl Books { } } +impl Books { + /// Restore a backup (admin INVARIANT 4; ledger_store::root INVARIANT + /// 7): a preview, or the books restored and opened. Books it replaces + /// are let go first, and opened again if the restore fails. + fn restore( + &self, + id: String, + source: String, + replace: bool, + confirmed: bool, + expect_head: Option<(u64, String)>, + ) -> Response { + // admin INVARIANT 3: never the root, nor the address given. + let said = |text: String| { + let text = if source.chars().count() > 1 { + text.replace(&source, "the backup given") + } else { + text + }; + Response::Refused(self.hide(text)) + }; + let id = match BookId::parse(&id) { + Ok(id) => id, + Err(e) => return said(e.to_string()), + }; + let _one = self.restoring.lock().unwrap_or_else(|e| e.into_inner()); + let git = match ledger_store::backup::Git::find() { + Ok(git) => git, + Err(e) => return said(e.to_string()), + }; + // A preview only reports (whether books of that id exist among + // what it says): the restore itself is refused over books, or + // replacing none. + if !confirmed { + return match self.root.restore_preview(&git, &source, &id) { + Ok(p) => Response::RestorePreview(admin::RestorePreview { + head: p.head, + blobs: p.blobs, + vault_files: p.vault_files, + exists: p.exists, + origin: p.origin, + }), + Err(e) => said(e.to_string()), + }; + } + let Some(expect_head) = expect_head else { + return said("a restore is confirmed against the preview's head: preview first".into()); + }; + let held = self.held().contains_key(id.as_str()); + if replace && !held { + return said(format!( + "there are no books called {:?} to replace: nothing was restored", + id.as_str() + )); + } + if !replace && held { + return said(format!( + "there are already books called {:?}: restore them under another id (--id), or replace them (--replace; they are kept in legacy/)", + id.as_str() + )); + } + // The books replaced are let go (their writer lock released), + // their backups idle until the restore is done: none of theirs + // reaches a target after the backup is read. + let old = if replace { + self.held + .write() + .unwrap_or_else(|e| e.into_inner()) + .remove(id.as_str()) + } else { + None + }; + let old_backups = match &old { + Some(Book::Open { backups, .. }) => Some(backups.clone()), + _ => None, + }; + let rt = tokio::runtime::Handle::current(); + // Its requests end first (a books.back_up_now among them waits for + // a run of the loop's), then its last run. + if let Some(Book::Open { store, .. }) = &old { + rt.block_on(store.let_go()); + } + let _paused = old_backups.as_ref().map(|b| rt.block_on(b.pause())); + match self.root.restore(&git, &source, &id, &expect_head, replace) { + Ok(r) => { + // Restored: kept whether or not it opens; its backups told + // whose books these were (backup.rs INVARIANTS 6, 7). + let book = self.open(&id); + let problem = book.problem(); + self.hold(&id, book); + Response::Restored(admin::Restored { + id: r.id.to_string(), + head: r.head, + blobs_verified: r.blobs_verified, + vault_files: r.vault_files, + replaced: r.replaced, + origin: r.origin, + problem, + }) + } + Err(e) => { + // The books it would have replaced, let go above, open + // again (only this restore runs, so nothing else holds them). + if replace && !self.held().contains_key(id.as_str()) { + let book = self.open(&id); + self.hold(&id, book); + } + said(e.to_string()) + } + } + } +} + /// The settings a settings file moves in at adoption, checked as /// `books.import_settings` checks them: they parse, they apply, and the /// vault key file it names (which the books don't keep) is the one given. @@ -1603,10 +1757,28 @@ pub fn run( args: impl IntoIterator, ) -> Result<(), Box> { let mut root: Option = None; + let mut backup_quiet = BACKUP_QUIET; + let mut drill_first = DRILL_FIRST; let mut args = args.into_iter(); while let Some(arg) = args.next() { match arg.as_str() { "--root" => root = Some(PathBuf::from(args.next().ok_or("--root needs a folder")?)), + // How long the books stay unwritten before a backup (tests). + "--backup-quiet" => { + let ms: u64 = args + .next() + .and_then(|v| v.parse().ok()) + .ok_or("--backup-quiet needs milliseconds")?; + backup_quiet = Duration::from_millis(ms); + } + // How soon after books open a drill is first looked for (tests). + "--drill-first" => { + let ms: u64 = args + .next() + .and_then(|v| v.parse().ok()) + .ok_or("--drill-first needs milliseconds")?; + drill_first = Duration::from_millis(ms); + } "--help" | "-h" => { print_usage(prog); return Ok(()); @@ -1684,6 +1856,9 @@ pub fn run( started: std::time::Instant::now(), principals: RwLock::new(Err(String::new())), session_tokens: Mutex::new(BTreeMap::new()), + backup_quiet, + drill_first, + restoring: Mutex::new(()), }); { let loaded = books.load_principals(); @@ -1753,6 +1928,10 @@ async fn serve( } let held: Vec = books.held().keys().cloned().collect(); eprintln!("wanderbooks server: ready, holding {} books", held.len()); + // Every open book is backed up when it opens, once its writing has + // stopped for a while, and again after a failed run (backup.rs). + let (backing, backup_stop) = (books.clone(), stop.clone()); + let backup_loop = tokio::spawn(async move { keep_backed_up(backing, backup_stop).await }); // Every open book's lapsed uploads go within SWEEP_EVERY of lapsing, // whatever else the server is doing (uploads.rs INVARIANT 3). let (sweeping, sweep_stop) = (books.clone(), stop.clone()); @@ -1823,6 +2002,22 @@ async fn serve( for store in books.stores() { store.close().await; } + // A last backup of what was written since the last one, for a while + // at most: the books are closing, nothing more is written. The loop + // stops at its next step (`stop`); a run of its still under way + // finishes first (each book runs one at a time). + drop(backup_loop); + let held: Vec> = books + .held() + .values() + .filter_map(|b| match b { + Book::Open { backups, .. } => Some(backups.clone()), + Book::Failed(_) => None, + }) + .collect(); + for b in held { + let _ = tokio::time::timeout(BACKUP_AT_CLOSE, b.run("close")).await; + } // Answers on their way out are sent, for a while: a client sending // a request it never finishes doesn't hold the books open. let answers = async { @@ -1916,10 +2111,288 @@ fn watch_for_close(stops: &Stops) -> std::io::Result<()> { Ok(()) } +/// What the backup loop knows of one open book. +struct Seen { + /// Its backups, by identity: books restored over others are new. + backups: Arc, + watch: Watch, +} + +/// One open book's backups, as the loop has watched them. +#[derive(Debug, Clone, Copy)] +struct Watch { + /// Its head's seq when last looked at, and when that changed. + seq: u64, + moved_at: Instant, + /// The seq last committed and offered to the targets. + backed: Option, + /// When the last run's commit failed, if it did: nothing runs again + /// for RETRY_AFTER. + failed_at: Option, + /// When the last run left a target failing, if it did: it is tried + /// again RETRY_AFTER on, written to or not. + target_failed_at: Option, + /// When a drill is next looked for. + drill_next: Instant, +} + +impl Watch { + /// Books first seen open now, at `seq`: a drill is first looked for + /// `drill_first` from now. + fn new(seq: u64, now: Instant, drill_first: Duration) -> Self { + Watch { + seq, + moved_at: now, + backed: None, + failed_at: None, + target_failed_at: None, + drill_next: now + drill_first, + } + } + + /// A run through `seq` came to `outcome` at `now`: None when its + /// commit failed, else whether every target holds it. + fn ran(&mut self, seq: u64, outcome: Option, now: Instant) { + match outcome { + None => self.failed_at = Some(now), + Some(all) => { + self.backed = Some(seq); + self.failed_at = None; + self.target_failed_at = (!all).then_some(now); + } + } + } +} + +/// Why a backup is due now, if one is: the books just seen open; or +/// written and quiet for `quiet` since (not yet backed up); or a target +/// left failing RETRY_AFTER ago. Nothing within RETRY_AFTER of a failed +/// commit. +fn backup_due( + seen: Option<&Watch>, + seq: u64, + now: Instant, + quiet: Duration, +) -> Option<&'static str> { + let Some(s) = seen else { + return Some("open"); + }; + let after = + |t: Option| t.is_some_and(|f| now.duration_since(f) >= crate::backup::RETRY_AFTER); + if s.failed_at.is_some() && !after(s.failed_at) { + return None; + } + let quiet_now = if s.seq == seq { + now.duration_since(s.moved_at) >= quiet + } else { + quiet.is_zero() + }; + if s.backed != Some(seq) && quiet_now { + return Some("write"); + } + after(s.target_failed_at).then_some("retry") +} + +/// Whether a run's every target holds the books: a target paused for +/// these books (restored as a copy of others) is no failure to try again. +fn every_target_held(status: &crate::backup::Status) -> bool { + status + .targets + .iter() + .all(|t| t.paused || t.last_error.is_none()) +} + +/// Whether to look for a drill due now (then again DRILL_LOOK on). +fn drill_look_due(seen: &Watch, now: Instant) -> bool { + now >= seen.drill_next +} + +/// Back every open book up (backup.rs) whenever [`backup_due`] says, and +/// drill its backups when a drill is due (looked for every DRILL_LOOK). +/// Until `stop`. +async fn keep_backed_up(books: Arc, stop: CancellationToken) { + let mut seen: BTreeMap = BTreeMap::new(); + let mut tick = tokio::time::interval(BACKUP_LOOK); + loop { + tokio::select! { + () = stop.cancelled() => return, + _ = tick.tick() => {} + } + let open: Vec<(String, Arc)> = books + .held() + .iter() + .filter_map(|(id, b)| match b { + Book::Open { backups, .. } => Some((id.clone(), backups.clone())), + Book::Failed(_) => None, + }) + .collect(); + seen.retain(|id, _| open.iter().any(|(o, _)| o == id)); + for (id, backups) in open { + if stop.is_cancelled() { + return; + } + let Some((seq, _)) = backups.head().await else { + continue; + }; + let now = Instant::now(); + // Books restored over others are seen anew. + if seen + .get(&id) + .is_some_and(|s| !Arc::ptr_eq(&s.backups, &backups)) + { + seen.remove(&id); + } + let due = backup_due( + seen.get(&id).map(|s| &s.watch), + seq, + now, + books.backup_quiet, + ); + let s = &mut seen + .entry(id) + .or_insert(Seen { + backups: backups.clone(), + watch: Watch::new(seq, now, books.drill_first), + }) + .watch; + if s.seq != seq { + s.seq = seq; + s.moved_at = now; + } + if let Some(why) = due { + let outcome = backups + .run(why) + .await + .ok() + .map(|status| every_target_held(&status)); + s.ran(seq, outcome, now); + } + if stop.is_cancelled() { + return; + } + // Drills (backup.rs INVARIANT 5). + if drill_look_due(s, now) { + s.drill_next = now + DRILL_LOOK; + if backups.drill_due().await { + backups.drill().await; + } + } + } + } +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn a_backup_is_due_on_open_then_once_quiet_after_a_write() { + let t0 = Instant::now(); + let quiet = Duration::from_secs(30); + let s = |seq, moved: u64, backed, failed: Option| Watch { + seq, + moved_at: t0 + Duration::from_secs(moved), + backed, + failed_at: failed.map(|f| t0 + Duration::from_secs(f)), + target_failed_at: None, + drill_next: t0, + }; + let at = |secs: u64| t0 + Duration::from_secs(secs); + // First seen: at once. + assert_eq!(backup_due(None, 5, t0, quiet), Some("open")); + // Backed up through the head: nothing. + assert_eq!( + backup_due(Some(&s(5, 0, Some(5), None)), 5, at(100), quiet), + None + ); + // Written: not until it has been quiet `quiet`. + assert_eq!( + backup_due(Some(&s(5, 0, Some(5), None)), 6, at(1), quiet), + None + ); + assert_eq!( + backup_due(Some(&s(6, 10, Some(5), None)), 6, at(39), quiet), + None + ); + assert_eq!( + backup_due(Some(&s(6, 10, Some(5), None)), 6, at(40), quiet), + Some("write") + ); + // No quiet asked for: at once. + assert_eq!( + backup_due(Some(&s(5, 0, Some(5), None)), 6, at(1), Duration::ZERO), + Some("write") + ); + // A failure waits RETRY_AFTER. + let retry = crate::backup::RETRY_AFTER.as_secs(); + let failed = s(6, 10, Some(5), Some(50)); + assert_eq!( + backup_due(Some(&failed), 6, at(50 + retry - 1), quiet), + None + ); + assert_eq!( + backup_due(Some(&failed), 6, at(50 + retry), quiet), + Some("write") + ); + // A drill is first looked for `drill_first` after the books are + // seen open, never at once. + let first = Duration::from_secs(60); + let w = Watch::new(1, t0, first); + assert!(!drill_look_due(&w, t0)); + assert!(!drill_look_due(&w, t0 + first - Duration::from_millis(1))); + assert!(drill_look_due(&w, t0 + first)); + assert_eq!((w.backed, w.failed_at, w.seq), (None, None, 1)); + // A run's outcome: its commit failed (nothing again for a + // while), or made and some target failing (tried again later, + // while writes still go to the others), or all well. + let mut w = Watch::new(1, t0, first); + w.ran(4, None, at(5)); + assert_eq!( + (w.backed, w.failed_at, w.target_failed_at), + (None, Some(at(5)), None) + ); + w.ran(4, Some(false), at(9)); + assert_eq!( + (w.backed, w.failed_at, w.target_failed_at), + (Some(4), None, Some(at(9))) + ); + w.seq = 5; + w.moved_at = at(10); + assert_eq!(backup_due(Some(&w), 5, at(10 + 30), quiet), Some("write")); + w.backed = Some(5); + assert_eq!(backup_due(Some(&w), 5, at(9 + retry - 1), quiet), None); + assert_eq!(backup_due(Some(&w), 5, at(9 + retry), quiet), Some("retry")); + // A paused target is no failure; another's error is. + let target = |paused: bool, error: Option<&str>| crate::backup::TargetStatus { + target: "repository".into(), + at: "x".into(), + last_ok_at: None, + through_seq: None, + stale: false, + last_error: error.map(str::to_string), + last_drill_at: None, + drill_error: None, + paused, + }; + let status = |targets| crate::backup::Status { + head_seq: 5, + targets, + note: None, + }; + assert!(every_target_held(&status(vec![ + target(true, Some("paused")), + target(false, None) + ]))); + assert!(!every_target_held(&status(vec![ + target(false, Some("no")), + target(false, None) + ]))); + assert!(every_target_held(&status(vec![]))); + w.ran(5, Some(true), at(9 + retry)); + assert_eq!((w.failed_at, w.target_failed_at), (None, None)); + assert_eq!(backup_due(Some(&w), 5, at(9 + retry * 3), quiet), None); + } + #[test] fn a_socket_pair_is_this_account_at_both_ends() { let (a, b) = std::os::unix::net::UnixStream::pair().unwrap(); diff --git a/crates/ledger-mcp/src/scopes.rs b/crates/ledger-mcp/src/scopes.rs index b5a3718..2025d9e 100644 --- a/crates/ledger-mcp/src/scopes.rs +++ b/crates/ledger-mcp/src/scopes.rs @@ -23,6 +23,11 @@ //! with include_tins, export_cpa_package, get_w9) need books:admin, //! and sealing one into the vault (set_tin, set_w9, which can replace //! the one there) needs books:post (review of slice 4). +//! 6. Where the books' backups go is books:admin's, as backing up is +//! (a backup is the whole book, its vault and key included): +//! setting a `backup.*` key, or importing settings that hold one, +//! needs books:admin; other settings books:post as before (review of +//! slice 7). //! ``` use ledger_store::auth::Scope; @@ -38,9 +43,17 @@ pub enum Needs { /// books:post for an approval rule (the control that gates posting), /// books:write for any other convention (INVARIANT 4). PostForApprovalRules, + /// books:admin for a change to a `backup.*` setting, books:post for + /// any other (INVARIANT 6). + AdminForBackupSettings, + /// As PostWhenConfirmed, and books:admin when the settings imported + /// hold a `[backup]` table (INVARIANT 6). + ImportSettings, } -use Needs::{Always, PostForApprovalRules, PostWhenConfirmed}; +use Needs::{ + AdminForBackupSettings, Always, ImportSettings, PostForApprovalRules, PostWhenConfirmed, +}; const R: Needs = Always(Scope::Read); const W: Needs = Always(Scope::Write); const P: Needs = Always(Scope::Post); @@ -49,12 +62,12 @@ const C: Needs = PostWhenConfirmed; const K: Needs = PostForApprovalRules; /// Every tool and what it needs (INVARIANT 1). -pub const TABLE: [(&str, Needs); 109] = [ +pub const TABLE: [(&str, Needs); 111] = [ ("books.handbook", R), ("books.start", R), // Settings are committed at once: the settings commit of the plan. - ("books.set_setting", P), - ("books.import_settings", C), + ("books.set_setting", AdminForBackupSettings), + ("books.import_settings", ImportSettings), ("books.export_settings", R), ("books.conventions", R), ("books.set_convention", K), @@ -166,6 +179,8 @@ pub const TABLE: [(&str, Needs); 109] = [ ("books.profit_and_loss_by_class", R), ("books.export_cpa_package", A), ("books.access_log", A), + ("books.backup", R), + ("books.back_up_now", A), ]; /// What `tool` needs, given its arguments; None for a tool this server @@ -178,11 +193,38 @@ pub fn needs( Some(match needs { Always(scope) => *scope, PostWhenConfirmed => { - let confirmed = args - .and_then(|a| a.get("confirmed")) - .and_then(serde_json::Value::as_bool) - .unwrap_or(false); - if confirmed { Scope::Post } else { Scope::Write } + if confirmed(args) { + Scope::Post + } else { + Scope::Write + } + } + AdminForBackupSettings => { + let backup = args + .and_then(|a| a.get("changes")) + .and_then(serde_json::Value::as_array) + .is_some_and(|changes| { + changes.iter().any(|c| { + c.get("key") + .and_then(serde_json::Value::as_str) + .is_some_and(|k| k.trim().to_ascii_lowercase().starts_with("backup.")) + }) + }); + if backup { Scope::Admin } else { Scope::Post } + } + ImportSettings => { + let backup = args + .and_then(|a| a.get("toml_text")) + .and_then(serde_json::Value::as_str) + .and_then(|t| t.parse::().ok()) + .is_some_and(|t| t.contains_key("backup")); + if backup { + Scope::Admin + } else if confirmed(args) { + Scope::Post + } else { + Scope::Write + } } PostForApprovalRules => { let kind = args @@ -197,15 +239,24 @@ pub fn needs( }) } +fn confirmed(args: Option<&serde_json::Map>) -> bool { + args.and_then(|a| a.get("confirmed")) + .and_then(serde_json::Value::as_bool) + .unwrap_or(false) +} + /// Whether a token with `scopes` may list `tool` (it can call it in some /// form: a tool that posts when confirmed shows to a books:write token, /// which may preview). pub fn shown(tool: &str, scopes: &std::collections::BTreeSet) -> bool { match TABLE.iter().find(|(name, _)| *name == tool) { Some((_, Always(scope))) => scopes.contains(scope), - Some((_, PostWhenConfirmed | PostForApprovalRules)) => { + Some((_, PostWhenConfirmed | PostForApprovalRules | ImportSettings)) => { scopes.contains(&Scope::Write) || scopes.contains(&Scope::Post) } + Some((_, AdminForBackupSettings)) => { + scopes.contains(&Scope::Post) || scopes.contains(&Scope::Admin) + } None => false, } } @@ -253,6 +304,56 @@ mod tests { } } + #[test] + fn where_backups_go_is_admins() { + let args = |v: serde_json::Value| v.as_object().cloned(); + let set = |keys: &[&str]| { + let changes: Vec<_> = keys + .iter() + .map(|k| json!({"key": k, "value": "x"})) + .collect(); + needs( + "books.set_setting", + args(json!({"changes": changes})).as_ref(), + ) + }; + assert_eq!(set(&["company.name"]), Some(Scope::Post)); + assert_eq!(set(&["backup.folder"]), Some(Scope::Admin)); + assert_eq!( + set(&["company.name", "backup.git_remote"]), + Some(Scope::Admin) + ); + assert_eq!(set(&[" Backup.Folder"]), Some(Scope::Admin)); + assert_eq!(needs("books.set_setting", None), Some(Scope::Post)); + let import = |text: &str, confirmed: bool| { + needs( + "books.import_settings", + args(json!({"toml_text": text, "confirmed": confirmed})).as_ref(), + ) + }; + assert_eq!( + import("[company]\nname = \"A\"\n", false), + Some(Scope::Write) + ); + assert_eq!(import("[company]\nname = \"A\"\n", true), Some(Scope::Post)); + assert_eq!( + import("[backup]\nfolder = \"/x\"\n", false), + Some(Scope::Admin) + ); + assert_eq!( + import("[backup]\nfolder = \"/x\"\n", true), + Some(Scope::Admin) + ); + // Shown to whoever can set some setting. + let post: std::collections::BTreeSet = [Scope::Post].into(); + let write: std::collections::BTreeSet = [Scope::Write].into(); + let admin: std::collections::BTreeSet = [Scope::Admin].into(); + assert!(shown("books.set_setting", &post)); + assert!(shown("books.set_setting", &admin)); + assert!(!shown("books.set_setting", &write)); + assert!(shown("books.import_settings", &write)); + } + #[test] fn a_tool_that_takes_confirmed_posts_only_when_confirmed() { // Exactly the tools whose arguments have `confirmed`. @@ -263,7 +364,7 @@ mod tests { .and_then(|p| p.as_object()) .is_some_and(|p| p.contains_key("confirmed")); let (_, needs) = TABLE.iter().find(|(n, _)| *n == tool.name).unwrap(); - assert_eq!(takes, *needs == C, "{}", tool.name); + assert_eq!(takes, matches!(*needs, C | ImportSettings), "{}", tool.name); let takes_kind = tool .input_schema .get("properties") diff --git a/crates/ledger-mcp/src/server.rs b/crates/ledger-mcp/src/server.rs index 4f2f348..49e685c 100644 --- a/crates/ledger-mcp/src/server.rs +++ b/crates/ledger-mcp/src/server.rs @@ -78,6 +78,9 @@ pub struct BooksServer { /// The files the books made for agents, waiting to be fetched (books /// served from the data root; over stdio a file comes back inline). exports: Option, + /// The books' backups (books served from the data root; none over + /// stdio). + backups: Option>, } /// Where a book's own files live around its data dir. @@ -146,7 +149,8 @@ impl Layout { /// whether or not anyone still waits for its answer. /// ``` pub struct SharedStore { - store: tokio::sync::Mutex, + /// None once let go (`let_go`): its books were replaced by a restore. + store: tokio::sync::Mutex>, stopped: std::sync::atomic::AtomicBool, /// Closing: no new request runs (INVARIANT 3). closing: std::sync::atomic::AtomicBool, @@ -160,6 +164,25 @@ pub struct SharedStore { } impl SharedStore { + /// Run `f` on the store under its lock, for a backup (backup.rs): + /// blocking. It reads only, and runs even while the books close (a + /// last backup), never after the server stopped. + pub fn with_store(&self, f: impl FnOnce(&Store) -> R) -> Result { + if self.stopped() { + return Err(STOPPED.to_string()); + } + let store = self.store.blocking_lock(); + let store = store.as_ref().ok_or(STOPPED)?; + Ok(f(store)) + } + + /// Run `f` on the store if no one holds it now (books just opened): + /// never waits, so never blocks a runtime's thread. + pub fn try_with_store(&self, f: impl FnOnce(&Store) -> R) -> Option { + let store = self.store.try_lock().ok()?; + store.as_ref().map(f) + } + /// An attachment's name, type and bytes (hash-verified), for a /// download: blocking (the store's lock, then the file). /// None when there is no such attachment. @@ -168,6 +191,7 @@ impl SharedStore { return Err(STOPPED.to_string()); } let store = self.store.blocking_lock(); + let store = store.as_ref().ok_or(STOPPED)?; let Some(meta) = store .list_attachments(None) .map_err(|e| e.to_string())? @@ -189,7 +213,7 @@ impl SharedStore { pub fn new(store: Store) -> Arc { Arc::new(Self { currency: store.base_currency(), - store: tokio::sync::Mutex::new(store), + store: tokio::sync::Mutex::new(Some(store)), stopped: std::sync::atomic::AtomicBool::new(false), closing: std::sync::atomic::AtomicBool::new(false), running: std::sync::atomic::AtomicUsize::new(0), @@ -220,6 +244,15 @@ impl SharedStore { self.stopped.load(std::sync::atomic::Ordering::SeqCst) } + /// Let go of the books for good (a restore replaces them): close as + /// [`Self::close`], stop, and drop the store, so its writer lock is + /// released while sessions still hold this. + pub async fn let_go(&self) { + self.close().await; + self.stop(); + self.store.lock().await.take(); + } + /// Stop taking requests (INVARIANT 1): after an internal error, or an /// access record that couldn't be written (access.rs INVARIANT 3). pub(crate) fn stop(&self) { @@ -275,7 +308,8 @@ const VAULT_IN_HISTORY: &str = "the TIN vault can't store anything here yet: the /// task is suspended. Dropped by a panic, it stops the server /// (SharedStore INVARIANT 1). pub struct StoreGuard<'a> { - guard: tokio::sync::MutexGuard<'a, Store>, + /// Holds a store (checked when it was taken). + guard: tokio::sync::MutexGuard<'a, Option>, shared: &'a SharedStore, /// The journal's head when the lock was taken. before: (u64, String), @@ -289,7 +323,10 @@ impl Drop for StoreGuard<'_> { } // A write while it was held: the head after it, for the call's // access record (taken under the same lock, so it is this call's). - let after = self.guard.head(); + let Some(store) = self.guard.as_ref() else { + return; + }; + let after = store.head(); if after != self.before { WROTE.with(|w| *w.borrow_mut() = Some(after)); } @@ -299,13 +336,13 @@ impl Drop for StoreGuard<'_> { impl std::ops::Deref for StoreGuard<'_> { type Target = Store; fn deref(&self) -> &Store { - &self.guard + self.guard.as_ref().expect("a guard holds a store") } } impl std::ops::DerefMut for StoreGuard<'_> { fn deref_mut(&mut self) -> &mut Store { - &mut self.guard + self.guard.as_mut().expect("a guard holds a store") } } @@ -598,6 +635,7 @@ impl BooksServer { access: None, uploads: None, exports: None, + backups: None, } } @@ -613,6 +651,12 @@ impl BooksServer { self } + /// Back the books up through `backups` (backup.rs). + pub fn with_backups(mut self, backups: Arc) -> Self { + self.backups = Some(backups); + self + } + /// Check every call against its caller's token, and record it in /// `log` (access.rs). pub fn with_access(mut self, log: Arc) -> Self { @@ -701,7 +745,11 @@ impl BooksServer { /// Every tool the server serves. pub(crate) fn tool_router() -> rmcp::handler::server::router::tool::ToolRouter { - Self::core_router() + Self::kept_router() + Self::access_router() + Self::files_router() + Self::core_router() + + Self::kept_router() + + Self::access_router() + + Self::files_router() + + Self::backup_router() } /// The settings in force for this request (reloaded at its start, in @@ -1052,21 +1100,25 @@ impl BooksServer { return Err(STOPPED.to_string()); } let guard = self.store.store.blocking_lock(); - // A request that panicked while this one waited stopped it. + // A request that panicked while this one waited stopped it; books + // let go hold no store. if self.store.stopped() { return Err(STOPPED.to_string()); } + let Some(store) = guard.as_ref() else { + return Err(STOPPED.to_string()); + }; // Books that keep their own settings: take any change (crate:: // settings INVARIANT 1). let timezone = { let mut live = self.live(); - live.sync(guard.ledger().settings(), self.store.currency.code()); + live.sync(store.ledger().settings(), self.store.currency.code()); live.current().timezone.clone() }; // "Today" from here on is the zone in force now (a change another // session made while this request waited included). TIMEZONE.with(|z| *z.borrow_mut() = timezone); - let before = guard.head(); + let before = store.head(); Ok(StoreGuard { guard, shared: &self.store, @@ -2523,6 +2575,8 @@ fn lines_out(lines: &[ledger_core::StatementLine]) -> Vec { #[path = "tools/access.rs"] mod access_tools; +#[path = "tools/backup.rs"] +mod backup_tools; #[path = "tools/files.rs"] mod files; #[path = "tools/kept.rs"] @@ -6982,7 +7036,7 @@ impl BooksServer { #[tool( name = "books.purge_tin", title = "Purge a TIN from the vault", - description = "Delete a party's full TIN and W-9 file from the TIN vault (after the retention period, at the payee's request, or to replace a wrong one), with the reason. Without confirmed=true it previews what the vault holds and changes nothing. Confirmed, the records are deleted for good and the books journal it: the party's TIN facts (party.tin_set: purged says when and why) and W-9 facts (party.w9_set: kept_at says so) keep the last four and drop the vault record. Works without the vault key. The next CPA package then lacks that TIN (and says so).", + description = "Delete a party's full TIN and W-9 file from the TIN vault (after the retention period, at the payee's request, or to replace a wrong one), with the reason. Without confirmed=true it previews what the vault holds and changes nothing. Confirmed, the records are deleted from the vault (backups made before keep their copies) and the books journal it: the party's TIN facts (party.tin_set: purged says when and why) and W-9 facts (party.w9_set: kept_at says so) keep the last four and drop the vault record. Works without the vault key. The next CPA package then lacks that TIN (and says so).", annotations( read_only_hint = false, destructive_hint = true, @@ -7048,7 +7102,7 @@ impl BooksServer { .or_else(|| party.tin.as_ref().map(|t| t.last4.clone())), journaled: false, note: format!( - "Confirming deletes {} for good{}. Make sure the CPA has what they need first.", + "Confirming deletes {}{}; backups made before keep their copies. Make sure the CPA has what they need first.", match (held_tin, held_w9) { (true, true) => "the TIN and the W-9 file", (true, false) => "the TIN", @@ -8831,8 +8885,11 @@ impl ServerHandler for BooksServer { .unwrap_or_default(); tools.retain(|t| crate::scopes::shown(&t.name, &scopes)); } else { - // Books served over stdio keep no access log: not offered. - tools.retain(|t| t.name != "books.access_log"); + // Books served over stdio keep no access log and aren't backed + // up here: not offered. + tools.retain(|t| { + !["books.access_log", "books.backup", "books.back_up_now"].contains(&&*t.name) + }); } let hints = context .protocol_version() @@ -8852,7 +8909,9 @@ impl ServerHandler for BooksServer { // the books' own when they keep them (never waiting for the books // here; a request holding them syncs them when it locks), else the // files'. - if let Ok(store) = self.store.store.try_lock() { + if let Ok(store) = self.store.store.try_lock() + && let Some(store) = store.as_ref() + { self.live() .sync(store.ledger().settings(), self.store.currency.code()); } @@ -9373,7 +9432,7 @@ mod tests { const D: Hints = (Some(false), Some(true), None); const WI: Hints = (Some(false), Some(false), Some(true)); const DI: Hints = (Some(false), Some(true), Some(true)); - let table: [(&str, Hints); 109] = [ + let table: [(&str, Hints); 111] = [ ("books.handbook", R), ("books.start", R), ("books.set_setting", DI), @@ -9483,6 +9542,8 @@ mod tests { ("books.profit_and_loss_by_class", R), ("books.export_cpa_package", W), ("books.access_log", R), + ("books.backup", R), + ("books.back_up_now", WI), ]; let tools = BooksServer::tool_router().list_all(); assert_eq!(tools.len(), table.len()); @@ -9514,11 +9575,12 @@ mod tests { for word in ["requiresuserinteraction", "requires_user_interaction"] { assert!(!json.contains(word), "{name} asks for user interaction"); } - // The books are a closed world: no tool reaches beyond them. + // The books are a closed world: no tool reaches beyond them, + // but the one that pushes a backup to the owner's targets. assert_eq!( a.open_world_hint, - Some(false), - "{name} has no openWorldHint false" + Some(*name == "books.back_up_now"), + "{name}: openWorldHint" ); } // docs/mcp-interface.md names the 28 destructive tools. @@ -9581,15 +9643,24 @@ mod tests { } assert!(n > 0, "{krate}: no error templates found"); } - // The files tool's refusals (and description): its sentences. - for line in include_str!("tools/files.rs").lines() { - let line = line.trim_start(); - if line.starts_with("//") { - continue; - } - for (i, text) in line.split('"').enumerate() { - if i % 2 == 1 && text.len() > 30 && text.contains(' ') { - out.push(("tools/files.rs text".into(), text.to_string())); + // The files and backup tools' refusals (and descriptions), and the + // backups' own words: their sentences. (What git itself says of a + // failed backup reaches the human as git said it: not ours.) + for (name, src) in [ + ("tools/files.rs", include_str!("tools/files.rs")), + ("tools/backup.rs", include_str!("tools/backup.rs")), + ("backup.rs", include_str!("backup.rs")), + ] { + let src = src.split("#[cfg(test)]").next().unwrap_or(src); + for line in src.lines() { + let line = line.trim_start(); + if line.starts_with("//") { + continue; + } + for (i, text) in line.split('"').enumerate() { + if i % 2 == 1 && text.len() > 30 && text.contains(' ') { + out.push((format!("{name} text"), text.to_string())); + } } } } diff --git a/crates/ledger-mcp/src/tools/backup.rs b/crates/ledger-mcp/src/tools/backup.rs new file mode 100644 index 0000000..ef5cb18 --- /dev/null +++ b/crates/ledger-mcp/src/tools/backup.rs @@ -0,0 +1,127 @@ +//! `books.backup` and `books.back_up_now` (plan 3.7, D-30): the books' +//! backups as they stand, and one made now. +//! +//! ```text +//! GOAL: the agent can tell the human whether the books are backed up, +//! and back them up now when asked. +//! SOURCE: plan 3.7 ("books.backup {status|run} reports it": two tools, +//! one read-only), D-30. +//! INVARIANTS: +//! 1. Only books served from the data root are backed up: over stdio +//! both tools are hidden, and refuse. +//! 2. A status reads the backup record only; a run makes and pushes a +//! backup (backup.rs) and answers the status after it. +//! ``` + +use super::*; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Default, Deserialize, JsonSchema)] +pub struct BackupParams {} + +#[derive(Debug, Default, Deserialize, JsonSchema)] +pub struct BackUpNowParams { + /// The backup folder's disk is new (or another): begin these books' + /// backup there again. Only when the human says so: a folder that + /// should hold one is otherwise a disk not connected. + #[serde(default)] + pub new_disk: bool, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct TargetOut { + /// "repository" (the private repository) or "folder". + pub target: String, + /// Where it is. + pub at: String, + /// When it last took a backup (UTC), if ever. + pub last_ok_at: Option, + /// The journal seq it holds, if any. + pub through_seq: Option, + /// More than a day without the books' latest: tell the human. + pub stale: bool, + /// Why its last backup failed, if it did. + pub last_error: Option, + /// When its backup was last proven restorable (UTC), if ever: each + /// is cloned and checked weekly. + pub last_drill_at: Option, + /// Why that check failed, if it did: tell the human. + pub drill_error: Option, + /// Not backed up there: these books were restored as a copy of others. + pub paused: bool, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct BackupOut { + /// The books' journal seq now. + pub head_seq: u64, + /// Each backup the settings name. + pub targets: Vec, + /// What to do, when none is set up. + pub note: Option, +} + +impl From for BackupOut { + fn from(s: crate::backup::Status) -> Self { + BackupOut { + head_seq: s.head_seq, + note: s.note, + targets: s + .targets + .into_iter() + .map(|t| TargetOut { + target: t.target, + at: t.at, + last_ok_at: t.last_ok_at, + through_seq: t.through_seq, + stale: t.stale, + last_error: t.last_error, + last_drill_at: t.last_drill_at, + drill_error: t.drill_error, + paused: t.paused, + }) + .collect(), + } + } +} + +impl BooksServer { + fn backups(&self) -> Result<&std::sync::Arc, String> { + self.backups.as_ref().ok_or_else(|| { + "these books aren't backed up by this server: only books kept in the books' own private folder are".to_string() + }) + } +} + +#[tool_router(router = backup_router, vis = "pub(super)")] +impl BooksServer { + #[tool( + name = "books.backup", + title = "Backups", + description = "Whether the books are backed up: each backup target the settings name (backup.git_remote, a private repository; backup.folder, a folder on another disk), when it last took the books, through which journal seq, and why it failed if it did. stale: it lacks the latest, and the first try to back that up was more than a day ago: tell the human. The books back themselves up a little after each change, when they open and before they close; books.back_up_now makes one now.", + annotations(read_only_hint = true, open_world_hint = false) + )] + async fn backup(&self, _params: Parameters) -> Result, String> { + Ok(Json(self.backups()?.status().await.into())) + } + + #[tool( + name = "books.back_up_now", + title = "Back up now", + description = "Back the books up now to every target the settings name, and answer how each went (as books.backup). new_disk: true only when the human says the backup folder's disk is new: the books' backup is then begun there again. Needs books:admin.", + annotations( + read_only_hint = false, + destructive_hint = false, + idempotent_hint = true, + open_world_hint = true + ) + )] + async fn back_up_now( + &self, + params: Parameters, + ) -> Result, String> { + let b = self.backups()?; + Ok(Json(b.run_with("asked", params.0.new_disk).await?.into())) + } +} diff --git a/crates/ledger-mcp/src/tools/kept.rs b/crates/ledger-mcp/src/tools/kept.rs index 1120d79..c572a14 100644 --- a/crates/ledger-mcp/src/tools/kept.rs +++ b/crates/ledger-mcp/src/tools/kept.rs @@ -803,6 +803,24 @@ impl BooksServer { "An open item is due within 30 days, or overdue: raise it with the human.", )); } + // The books' backups (backup.rs): none set up, or one behind or + // failing. + if let Some(b) = &self.backups { + let st = b.standing(store); + if st.note.is_some() { + next_steps.push(step( + "books.backup", + None, + "No backups are set up: ask the human for a private repository and a folder on another disk, then set backup.git_remote and backup.folder (books.set_setting).", + )); + } else if st.needs_attention() { + next_steps.push(step( + "books.backup", + None, + "A backup is behind or failing: books.backup says which and why; tell the human.", + )); + } + } match focus { "close" => next_steps.push(step( "books.conventions", diff --git a/crates/ledger-mcp/src/types.rs b/crates/ledger-mcp/src/types.rs index 41cb566..7a5d23c 100644 --- a/crates/ledger-mcp/src/types.rs +++ b/crates/ledger-mcp/src/types.rs @@ -3383,7 +3383,7 @@ pub struct ReconciliationsOut { /// Parameters for `books.handbook`. #[derive(Debug, Default, Deserialize, JsonSchema)] pub struct HandbookParams { - /// The section to read: contract, entries, drafts-and-reversals, evidence-and-receipts, attachments, files-in-out, reconciliation, fixed-assets, prepaid-and-recurring, revenue-and-payouts, parties-and-tins, 1099, cpa-handoff, settings-and-conventions, viewer, rules-that-change, control-roles, tracing-figures, or "all" for the whole handbook in order. Omit for the index. + /// The section to read: contract, entries, drafts-and-reversals, evidence-and-receipts, attachments, files-in-out, reconciliation, fixed-assets, prepaid-and-recurring, revenue-and-payouts, parties-and-tins, 1099, cpa-handoff, settings-and-conventions, viewer, backups, rules-that-change, control-roles, tracing-figures, or "all" for the whole handbook in order. Omit for the index. #[serde(default)] pub section: Option, } diff --git a/crates/ledger-store/src/backup.rs b/crates/ledger-store/src/backup.rs new file mode 100644 index 0000000..a945f0a --- /dev/null +++ b/crates/ledger-store/src/backup.rs @@ -0,0 +1,2092 @@ +//! Backups (plan 3.7, D-30, D-31): the book as it is, in git. Each book +//! keeps a bare repository, `archive.git`, whose commits are its journal, +//! its stored files, its vault and keys, and its access log as they were; +//! it is pushed to the owner's private repository and to a repository in +//! the owner's folder. Not encrypted (D-31): the destinations keep it +//! private. +//! +//! ```text +//! GOAL: a book's every write survives the loss of this computer, and a +//! backup is proven whole before it is trusted or restored. +//! SOURCE: plan 3.7 ("Each run: verifies the chain; commits the new +//! journal lines and blobs to archive.git; pushes to the +//! targets"; "Proven before use"; drills), D-30 (both targets), +//! D-31 (no encryption). The chain isn't verified at each run: the +//! server verifies it when it opens the books, every write after +//! goes through the store, and drills prove what reached each +//! target. +//! INVARIANTS: +//! 1. A commit holds exactly the files under the backed-up paths +//! (BACKED_UP) of the book's folder as they are at that moment: every +//! one, by name (checked against the folder after staging; no ignore +//! rule applies), and byte for byte (no attributes rule can reach it: +//! none of the owner's or the system's is read, the archive's own is +//! taken out, and a `.gitattributes` in the book's folder refuses the +//! backup). The caller holds the store's lock, so the journal is whole +//! lines and the vault matches it. A commit with nothing changed is +//! not made. +//! 2. A push only moves a target forward (fast-forward): one that has +//! diverged is reported, never forced. +//! 3. A backup is trusted only after it is cloned, its tree found to be +//! plain files under BACKED_UP only (no link, no submodule, nothing +//! else) before any of it is written out, its journal's chain +//! replayed, every stored file re-hashed, every TIN record the books +//! name present with the vault's key file, and its head found +//! equal to the one its commit records (`verify`); a drill also +//! checks its journal is a byte prefix of the live one. +//! 4. git runs with its configuration pinned where it matters: no system +//! config or attributes; no hooks, here or in a repository on this +//! computer it pushes to; no prompts, signing, ignore or attributes +//! files, file watchers or command-running transports; our own +//! identity, the files ref format. A repository on this computer is +//! pushed to or read only if plain: a bare repository at that very +//! path, nothing in it sending git elsewhere (a `.git`, `commondir`, +//! `gitdir`, objects borrowed from elsewhere, a link where it or its +//! own parts are, something beside it git would take instead), its configuration only +//! what `git init --bare` writes and where a copy of one came from; +//! and it is read as a far one is. A git +//! that says nothing for GIT_TIMEOUT is stopped with all it started. +//! What git said of itself is kept in whole lines; callers hide the +//! paths in it before an agent sees it (ledger-mcp backup.rs, +//! root.rs). +//! ``` + +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +/// A book's backup repository, in its own folder. +pub const ARCHIVE_DIR: &str = "archive.git"; +/// What a backup holds, of a book's folder (INVARIANT 1). +pub const BACKED_UP: [&str; 5] = [ + crate::JOURNAL_FILE, + "blobs", + crate::root::VAULT_DIR, + crate::root::KEYS_DIR, + crate::access::ACCESS_FILE, +]; +/// The longest a git command may take before it is stopped. +pub const GIT_TIMEOUT: Duration = Duration::from_secs(300); + +/// Why a backup couldn't be made, pushed or trusted. Never names a path of +/// the data root (INVARIANT 4). +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum BackupError { + /// git isn't installed. + #[error("git isn't installed: backups need it (install git, then try again)")] + NoGit, + /// A git command failed; what it said, in whole lines (paths in it are + /// the caller's to hide, INVARIANT 4). + #[error("git {0}: {1}")] + Git(String, String), + /// A git command ran past its time limit. + #[error("git {0} took too long and was stopped")] + TimedOut(String), + /// A folder target that isn't there (an external disk not mounted). + #[error("the backup folder isn't there (is its disk connected?)")] + NoFolder, + /// A folder target that took these books' backups but holds none now + /// (a disk not mounted on its mount point, or a new disk). + #[error( + "the backup folder holds no backup of these books: connect the disk that does; if it is a new disk, back up now saying so (new_disk)" + )] + NoBackupThere, + /// The target moved on without us: never forced (INVARIANT 2). + #[error("the backup at {0} has changes this computer doesn't: it was not overwritten")] + Diverged(String), + /// A backup that isn't whole. + #[error("that backup can't be trusted: {0}")] + Damaged(String), + /// The disk refused. + #[error("the backup couldn't be made: {0}")] + Io(std::io::ErrorKind), +} + +impl From for BackupError { + fn from(e: std::io::Error) -> Self { + BackupError::Io(e.kind()) + } +} + +/// The git program, found once. +#[derive(Debug, Clone)] +pub struct Git { + exe: PathBuf, + /// How long it may say nothing before it is stopped (GIT_TIMEOUT). + quiet: Duration, +} + +impl Git { + /// git, from the usual places (never the working directory). + pub fn find() -> Result { + let dirs = std::env::var_os("PATH") + .map(|p| std::env::split_paths(&p).collect::>()) + .unwrap_or_default(); + dirs.iter() + .chain( + ["/usr/bin", "/bin", "/usr/local/bin", "/opt/homebrew/bin"] + .iter() + .map(PathBuf::from) + .collect::>() + .iter(), + ) + .filter(|d| d.is_absolute()) + .map(|d| d.join(if cfg!(windows) { "git.exe" } else { "git" })) + .find(|p| p.is_file()) + .map(|exe| Git { + exe, + quiet: GIT_TIMEOUT, + }) + .ok_or(BackupError::NoGit) + } + + /// A git command with a clean configuration (INVARIANT 4). + fn command(&self) -> Command { + let mut c = Command::new(&self.exe); + c.env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_ATTR_NOSYSTEM", "1") + .env("GIT_TERMINAL_PROMPT", "0") + .env("GIT_ASKPASS", "true") + .env_remove("GIT_DIR") + .env_remove("GIT_WORK_TREE") + .env_remove("GIT_INDEX_FILE") + .args([ + "-c", + "core.hooksPath=/dev/null", + "-c", + "commit.gpgsign=false", + "-c", + "user.name=wanderbooks", + "-c", + "user.email=backup@wanderbooks.invalid", + "-c", + "init.defaultBranch=main", + "-c", + "core.autocrlf=false", + // The owner's own ignore and attributes files never leave a + // file out of a backup or change its bytes; nothing runs + // to watch the files. + "-c", + "core.excludesFile=/dev/null", + "-c", + "core.attributesFile=/dev/null", + "-c", + "core.fsmonitor=false", + "-c", + "init.templateDir=", + "-c", + "init.defaultRefFormat=files", + "-c", + "init.defaultObjectFormat=sha1", + "-c", + "log.showSignature=false", + "-c", + "push.gpgSign=false", + // A link is never made from a backup's tree (verify + // refuses one before checkout, too). + "-c", + "core.symlinks=false", + // Only these ways of reaching a repository: never one that + // runs a command (ext::, fd::), whatever the owner's own + // configuration allows. + "-c", + "protocol.allow=never", + "-c", + "protocol.ext.allow=never", + "-c", + "protocol.fd.allow=never", + "-c", + "protocol.file.allow=always", + "-c", + "protocol.ssh.allow=always", + "-c", + "protocol.https.allow=always", + ]) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + c + } + + /// Run `args` (named `what` in errors): its output. Stopped, with every + /// process it started, once it has said nothing for GIT_TIMEOUT + /// (a push or a clone reports its progress as it goes). + fn run(&self, what: &str, args: &[&std::ffi::OsStr]) -> Result { + use std::sync::atomic::{AtomicU64, Ordering}; + let mut c = self.command(); + c.args(args); + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + c.process_group(0); + } + let mut child = c.spawn().map_err(|e| match e.kind() { + std::io::ErrorKind::NotFound => BackupError::NoGit, + kind => BackupError::Io(kind), + })?; + let started = Instant::now(); + // Milliseconds since `started` when git last said anything. + let heard = std::sync::Arc::new(AtomicU64::new(0)); + let drain = |pipe: Option>, cap: usize| { + let (tx, rx) = std::sync::mpsc::channel(); + let heard = heard.clone(); + std::thread::spawn(move || { + let mut bytes = Vec::new(); + if let Some(mut p) = pipe { + let mut buf = [0u8; 8192]; + while let Ok(n) = p.read(&mut buf) { + if n == 0 { + break; + } + heard.store(started.elapsed().as_millis() as u64, Ordering::Relaxed); + // Bounded: a git that says this much is kept only + // in part, its last part (where its errors are). + bytes.extend_from_slice(&buf[..n]); + if bytes.len() > cap.saturating_mul(2) { + bytes.drain(..bytes.len() - cap); + } + } + } + if bytes.len() > cap { + bytes.drain(..bytes.len() - cap); + } + let _ = tx.send(bytes); + }); + rx + }; + use std::io::Read; + // What it answers is kept whole (a listing of every file); what it + // says of itself, in part. + let stdout = drain(child.stdout.take().map(|p| Box::new(p) as _), usize::MAX); + let stderr = drain(child.stderr.take().map(|p| Box::new(p) as _), OUTPUT_MAX); + // Git and every process it started (its own group, above). + #[cfg(unix)] + // SAFETY: kill has no preconditions; the group is git's own + // (process_group(0) above), so this stops it and what it started. + let kill_group = |pid: u32| unsafe { + libc::kill(-(pid as libc::pid_t), libc::SIGKILL); + }; + #[cfg(not(unix))] + let kill_group = |_: u32| {}; + let status = loop { + if let Some(status) = child.try_wait()? { + break status; + } + let quiet_for = started + .elapsed() + .saturating_sub(Duration::from_millis(heard.load(Ordering::Relaxed))); + if quiet_for > self.quiet { + kill_group(child.id()); + let _ = child.kill(); + let _ = child.wait(); + return Err(BackupError::TimedOut(what.to_string())); + } + std::thread::sleep(Duration::from_millis(20)); + }; + // What it started may still hold its pipes: not waited for long. + kill_group(child.id()); + let wait = Duration::from_secs(10); + let stdout = stdout.recv_timeout(wait).unwrap_or_default(); + let stderr = stderr.recv_timeout(wait).unwrap_or_default(); + if status.success() { + Ok(String::from_utf8_lossy(&stdout).trim().to_string()) + } else { + let cut = stderr.len() >= OUTPUT_MAX; + Err(BackupError::Git(what.to_string(), said(&stderr, cut))) + } + } +} + +/// The most kept of what a git says of itself. +const OUTPUT_MAX: usize = 1 << 20; + +/// What git said when it failed: its messages without its progress, the +/// last whole lines of it (never a line cut part way, which could hide a +/// path from those who hide paths, INVARIANT 4); `cut` when only the last +/// of what it said was kept (its first line may be part of one). +fn said(stderr: &[u8], cut: bool) -> String { + let text = String::from_utf8_lossy(stderr); + let text = match (cut, text.find(['\r', '\n'])) { + (true, Some(at)) => &text[at + 1..], + (true, None) => "", + (false, _) => &text[..], + }; + let lines: Vec<&str> = text + .split(['\r', '\n']) + .map(str::trim) + .filter(|l| !l.is_empty() && !l.contains("% (") && !l.ends_with(", done.")) + .collect(); + let mut kept: Vec<&str> = Vec::new(); + let mut size = 0; + for line in lines.iter().rev() { + if line.len() > 1000 || size + line.len() > 1000 { + break; + } + size += line.len() + 1; + kept.push(line); + } + kept.reverse(); + kept.join("\n") +} + +/// One backup made: its commit and the journal head it holds. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Made { + /// The commit's id. + pub commit: String, + /// The journal's head it holds: seq and hash. + pub head: (u64, String), +} + +fn os(s: &str) -> &std::ffi::OsStr { + std::ffi::OsStr::new(s) +} + +/// The archive of the book in `book_dir`, made (bare, private) if missing. +/// No attributes rule of its own (which could change the bytes it keeps) +/// is ever left in it. +fn archive(git: &Git, book_dir: &Path) -> Result { + let dir = book_dir.join(ARCHIVE_DIR); + if !dir.join("HEAD").is_file() { + crate::root::private_book_folder(book_dir, ARCHIVE_DIR) + .map_err(|e| BackupError::Damaged(e.to_string()))?; + git.run( + "init", + &[os("init"), os("--bare"), os("-q"), dir.as_os_str()], + )?; + } + let _ = std::fs::remove_file(dir.join("info").join("attributes")); + // Nor one in the book's folder: refused, never followed. + if book_dir.join(".gitattributes").exists() { + return Err(BackupError::Damaged( + "the book's folder holds a file (.gitattributes) no book keeps".into(), + )); + } + Ok(dir) +} + +/// Commit the book in `book_dir`, whose journal's head is `head`, to its +/// archive (INVARIANT 1): the commit, or None when nothing changed since +/// the last. The caller holds the store's lock while this runs. +pub fn commit( + git: &Git, + book_dir: &Path, + book: &str, + head: (u64, String), +) -> Result, BackupError> { + let archive = archive(git, book_dir)?; + clear_old_locks(&archive); + let git_dir = format!("--git-dir={}", archive.display()); + let work_tree = format!("--work-tree={}", book_dir.display()); + let paths: Vec<&str> = BACKED_UP + .iter() + .copied() + .filter(|p| book_dir.join(p).exists()) + .collect(); + if !paths.is_empty() { + let mut add = vec![ + os(&git_dir), + os(&work_tree), + os("add"), + os("-A"), + os("-f"), + os("--"), + ]; + add.extend(paths.iter().map(|p| os(p))); + git.run("add", &add)?; + } + // What was backed up and is now gone altogether (a vault purged to + // nothing) goes from the next commit too. + let gone: Vec<&str> = BACKED_UP + .iter() + .copied() + .filter(|p| !book_dir.join(p).exists()) + .collect(); + if !gone.is_empty() { + let mut rm = vec![ + os(&git_dir), + os(&work_tree), + os("rm"), + os("-r"), + os("-q"), + os("--cached"), + os("--ignore-unmatch"), + os("--"), + ]; + rm.extend(gone.iter().map(|p| os(p))); + git.run("rm", &rm)?; + } + // Every file there is in it, nothing else (INVARIANT 1). + let listed = git.run( + "ls-files", + &[os(&git_dir), os(&work_tree), os("ls-files"), os("-z")], + )?; + let mut indexed: Vec = listed + .split('\0') + .filter(|p| !p.is_empty()) + .map(str::to_string) + .collect(); + indexed.sort(); + let mut on_disk = Vec::new(); + for p in &paths { + files_under(book_dir, Path::new(p), &mut on_disk)?; + } + on_disk.sort(); + if on_disk + .iter() + .any(|p| p.rsplit('/').next() == Some(".gitattributes")) + { + return Err(BackupError::Damaged( + "the book's folder holds a file (.gitattributes) no book keeps".into(), + )); + } + if indexed != on_disk { + return Err(BackupError::Damaged( + "the backup wouldn't hold the books exactly as they are".into(), + )); + } + let unchanged = git + .run( + "diff", + &[ + os(&git_dir), + os(&work_tree), + os("diff"), + os("--cached"), + os("--quiet"), + ], + ) + .is_ok(); + let has_commits = git + .run( + "rev-parse", + &[ + os(&git_dir), + os("rev-parse"), + os("-q"), + os("--verify"), + os("HEAD"), + ], + ) + .is_ok(); + if unchanged && has_commits { + return Ok(None); + } + // The books it holds by name too (a restore tells their own from + // others'). + let message = format!("seq {} head {}\n\nbook {book}", head.0, head.1); + git.run( + "commit", + &[ + os(&git_dir), + os(&work_tree), + os("commit"), + os("-q"), + os("--no-verify"), + os("--allow-empty"), + os("-m"), + os(&message), + ], + )?; + let commit = git.run("rev-parse", &[os(&git_dir), os("rev-parse"), os("HEAD")])?; + Ok(Some(Made { commit, head })) +} + +/// Stage the book's stored files in its archive ahead of a commit, outside +/// the store's lock: they never change once written (named by their +/// hash), so the commit under the lock has only the new ones to read. +pub fn stage_blobs(git: &Git, book_dir: &Path) -> Result<(), BackupError> { + let blobs = crate::store::BLOBS_DIR; + if !book_dir.join(blobs).is_dir() { + return Ok(()); + } + let archive = archive(git, book_dir)?; + clear_old_locks(&archive); + let git_dir = format!("--git-dir={}", archive.display()); + let work_tree = format!("--work-tree={}", book_dir.display()); + git.run( + "add", + &[ + os(&git_dir), + os(&work_tree), + os("add"), + os("-A"), + os("-f"), + os("--"), + os(blobs), + ], + )?; + Ok(()) +} + +/// The locks a git left behind (stopped, the server killed) would refuse +/// every later backup: cleared once older than any git may run quiet, +/// so never one a git still at work holds. +fn clear_old_locks(archive: &Path) { + for lock in ["index.lock", "HEAD.lock", "refs/heads/main.lock"] { + let path = archive.join(lock); + let old = std::fs::metadata(&path) + .and_then(|m| m.modified()) + .is_ok_and(|t| t.elapsed().is_ok_and(|e| e > GIT_TIMEOUT)); + if old { + let _ = std::fs::remove_file(path); + } + } +} + +/// Every file under `rel` in `base` (or `rel` itself), as paths relative to +/// `base` with `/` between their parts. +fn files_under(base: &Path, rel: &Path, out: &mut Vec) -> Result<(), BackupError> { + let full = base.join(rel); + let meta = std::fs::symlink_metadata(&full)?; + if meta.is_dir() { + for entry in std::fs::read_dir(&full)? { + files_under(base, &rel.join(entry?.file_name()), out)?; + } + } else { + let parts: Vec = rel + .components() + .map(|c| c.as_os_str().to_string_lossy().into_owned()) + .collect(); + out.push(parts.join("/")); + } + Ok(()) +} + +/// What `git init --bare` writes in a repository's configuration, and +/// nothing else a backup's repository may hold: a setting of its own (a +/// command to run, objects borrowed from elsewhere, a work tree to write) +/// could make a push or a clone do more than move the books. +const PLAIN_SETTINGS: [&str; 9] = [ + "core.repositoryformatversion", + "core.filemode", + "core.bare", + "core.logallrefupdates", + "core.ignorecase", + "core.precomposeunicode", + "core.symlinks", + "extensions.objectformat", + "extensions.refstorage", +]; + +/// Refuse the repository on this computer at `address` unless it is plain: +/// its configuration only what `git init --bare` writes, no objects of +/// another's. +fn plain_repository(git: &Git, address: &str) -> Result<(), BackupError> { + let dir = PathBuf::from(address.strip_prefix("file://").unwrap_or(address)); + let not_plain = || { + BackupError::Damaged( + "its repository holds settings of its own: back up to one made only for these books" + .into(), + ) + }; + // A bare repository right there, as git will find it: nothing that + // sends git somewhere else first (a .git in it, another repository's + // files named in it), and not a name git would look beside. + // git takes it as a repository (and so looks no further, beside it) + // only when it holds objects, refs and a HEAD naming one. + let head = std::fs::read_to_string(dir.join("HEAD")).unwrap_or_default(); + let head = head.trim(); + let names_one = head.strip_prefix("ref: refs/").is_some() + || (head.len() >= 40 && head.bytes().all(|b| b.is_ascii_hexdigit())); + if !names_one + || !dir.join("config").is_file() + || !dir.join("objects").is_dir() + || !dir.join("refs").is_dir() + { + return Err(BackupError::Damaged( + "that isn't a backup's repository: name the repository itself".into(), + )); + } + for elsewhere in [".git", "commondir", "gitdir"] { + if std::fs::symlink_metadata(dir.join(elsewhere)).is_ok() { + return Err(not_plain()); + } + } + // Nor a link where its own parts are (which would send what is + // written there elsewhere); the folder holding it may be one. + let trimmed = PathBuf::from( + address + .strip_prefix("file://") + .unwrap_or(address) + .trim_end_matches('/'), + ); + for part in [ + trimmed.clone(), + dir.join("HEAD"), + dir.join("config"), + dir.join("objects"), + dir.join("refs"), + ] { + if std::fs::symlink_metadata(&part).is_ok_and(|m| m.file_type().is_symlink()) { + return Err(not_plain()); + } + } + // Nor anything git would take in its place, beside it. + let bare = address + .strip_prefix("file://") + .unwrap_or(address) + .trim_end_matches('/'); + for beside in [".git", ".bundle"] { + if std::fs::symlink_metadata(format!("{bare}{beside}")).is_ok() { + return Err(not_plain()); + } + } + if dir.join("objects").join("info").join("alternates").exists() { + return Err(not_plain()); + } + let config = dir.join("config"); + let file = format!("--file={}", config.display()); + let names = git.run( + "config", + &[os("config"), os(&file), os("--list"), os("--name-only")], + )?; + // A copy made with `git clone --bare` also names where it came from: + // only that. + let plain = |name: &str| { + let name = name.trim().to_ascii_lowercase(); + let tail = |section: &str, keys: &[&str]| { + name.strip_prefix(section) + .and_then(|rest| rest.rsplit_once('.')) + .is_some_and(|(_, key)| keys.contains(&key)) + }; + PLAIN_SETTINGS.contains(&name.as_str()) + || tail("remote.", &["url", "fetch", "mirror"]) + || tail("branch.", &["remote", "merge"]) + // Made with `git init --bare --shared`: harmless. + || name == "core.sharedrepository" + || name == "receive.denynonfastforwards" + }; + if names.lines().any(|n| !plain(n)) { + return Err(not_plain()); + } + Ok(()) +} + +/// Whether git reads `address` as a repository on this computer (a path, +/// or `file://`), not one it reaches over the network (`ssh://`, +/// `https://`, or `host:path`). +pub fn is_local(address: &str) -> bool { + if let Some((scheme, _)) = address.split_once("://") { + return scheme == "file"; + } + match address.split_once(':') { + // host:path, where the host part holds no slash. + Some((host, _)) => host.contains('/'), + None => true, + } +} + +/// Where a backup goes (D-30). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Target { + /// The owner's private repository, as git names it. + Remote(String), + /// The owner's folder: a repository in it named `.git`. + Folder(PathBuf), +} + +impl Target { + /// Where the backup of the book `book` is, as git reads it. + pub fn source(&self, book: &str) -> String { + match self { + Target::Remote(url) => url.clone(), + Target::Folder(folder) => folder.join(format!("{book}.git")).display().to_string(), + } + } + + /// What it is called in records: "repository" or "folder". + pub fn kind(&self) -> &'static str { + match self { + Target::Remote(_) => "repository", + Target::Folder(_) => "folder", + } + } +} + +/// Push the archive of the book `book` in `book_dir` to `target` +/// (INVARIANT 2): the commit it now holds. +/// +/// A folder target's repository is made only when `may_create` (it never +/// took a backup): one that took backups and is gone is a disk not +/// connected (a mount point with nothing mounted on it), never made again +/// where the disk should be. +pub fn push( + git: &Git, + book_dir: &Path, + book: &str, + target: &Target, + may_create: bool, +) -> Result { + let archive = archive(git, book_dir)?; + let git_dir = format!("--git-dir={}", archive.display()); + let to = match target { + // Never read as an option, never a command (protocols, above). + Target::Remote(url) if url.starts_with('-') || url.trim().is_empty() => { + return Err(BackupError::Damaged( + "that isn't a repository address git takes".into(), + )); + } + Target::Remote(url) => url.clone(), + Target::Folder(folder) => { + if !folder.is_dir() { + return Err(BackupError::NoFolder); + } + let repo = folder.join(format!("{book}.git")); + if !repo.join("HEAD").is_file() { + if !may_create { + return Err(BackupError::NoBackupThere); + } + git.run( + "init", + &[os("init"), os("--bare"), os("-q"), repo.as_os_str()], + )?; + } + repo.display().to_string() + } + }; + if matches!(target, Target::Folder(_)) || is_local(&to) { + plain_repository(git, &to)?; + } + // A repository on this computer is updated by a git we start, which + // doesn't get our configuration: told, in its own words, to run no + // hooks (a folder others can write could hold some). + let exe = git.exe.display().to_string(); + let receive = format!("--receive-pack='{exe}' -c core.hooksPath=/dev/null receive-pack"); + // Its progress keeps it from counting as stalled (Git::run). + let mut args = vec![ + os(&git_dir), + os("push"), + os("--progress"), + os("--no-verify"), + ]; + if matches!(target, Target::Folder(_)) || is_local(&to) { + if exe.contains('\'') { + return Err(BackupError::Damaged( + "git is installed where its hooks can't be turned off (a quote in its path)".into(), + )); + } + args.push(os(&receive)); + } + args.extend([os(&to), os("HEAD:refs/heads/main")]); + match git.run("push", &args) { + Ok(_) => {} + Err(BackupError::Git(_, said)) + if said.contains("non-fast-forward") + || said.contains("[rejected]") + || said.contains("fetch first") => + { + return Err(BackupError::Diverged(match target { + Target::Remote(url) => url.clone(), + Target::Folder(_) => "the backup folder".to_string(), + })); + } + Err(e) => return Err(e), + } + git.run("rev-parse", &[os(&git_dir), os("rev-parse"), os("HEAD")]) +} + +/// A backup proven whole (INVARIANT 3). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Verified { + /// Its journal's head: seq and hash. + pub head: (u64, String), + /// Stored files re-hashed. + pub blobs: u64, + /// Where it was cloned (the caller's scratch folder), its working tree + /// the book as it was. + pub tree: PathBuf, + /// The id of the books it holds, as their last backup named them. + pub book: Option, +} + +/// Clone the backup at `source` (a repository's address, or its folder: +/// `.git` in a backup folder) +/// into `scratch` (a new folder) and prove it whole (INVARIANT 3). +pub fn verify(git: &Git, source: &str, scratch: &Path) -> Result { + if scratch.exists() { + return Err(BackupError::Damaged("its scratch folder is in use".into())); + } + // One on this computer is read as one far away would be (so it reports + // its progress), once it is known to run nothing of its own. + if is_local(source) { + plain_repository(git, source)?; + } + git.run( + "clone", + &[ + os("clone"), + os("--progress"), + os("--no-local"), + os("--no-checkout"), + os("--branch"), + os("main"), + os("--"), + os(source), + scratch.as_os_str(), + ], + )?; + let git_dir = format!("--git-dir={}", scratch.join(".git").display()); + // Its tree before any of it is written out: plain files, only where a + // book keeps what is backed up. A link or a submodule (which could + // point the checkout, or the store opened on it, anywhere) or anything + // else is refused. + let listed = git.run( + "ls-tree", + &[ + os(&git_dir), + os("ls-tree"), + os("-r"), + os("-z"), + os("--full-tree"), + os("HEAD"), + ], + )?; + for entry in listed.split('\0').filter(|e| !e.is_empty()) { + let (meta, path) = entry.split_once('\t').unwrap_or((entry, "")); + let mode = meta.split(' ').next().unwrap_or(""); + let top = path.split('/').next().unwrap_or(""); + if !["100644", "100755"].contains(&mode) || !BACKED_UP.contains(&top) { + return Err(BackupError::Damaged(format!( + "it holds {path:?}, which no book keeps" + ))); + } + } + let work_tree = format!("--work-tree={}", scratch.display()); + git.run( + "checkout", + &[ + os(&git_dir), + os(&work_tree), + os("checkout"), + os("-q"), + os("-f"), + os("main"), + ], + )?; + let mut store = crate::Store::open(scratch).map_err(|e| BackupError::Damaged(e.to_string()))?; + store + .verify_and_heal() + .map_err(|e| BackupError::Damaged(e.to_string()))?; + let blobs = store + .verify_blobs() + .map_err(|e| BackupError::Damaged(e.to_string()))?; + let head = store.head(); + // Every TIN record the books name is there, and the key that opens + // them. + let named: Vec = store + .ledger() + .parties() + .flat_map(|p| { + [ + p.tin.as_ref().and_then(|t| t.vault_record.clone()), + p.w9.as_ref().and_then(|w| w.vault_record.clone()), + ] + }) + .flatten() + .collect(); + drop(store); + if !named.is_empty() { + use sha2::{Digest, Sha256}; + let vault = scratch.join(crate::root::VAULT_DIR); + let mut held = std::collections::BTreeSet::new(); + if vault.is_dir() { + for f in std::fs::read_dir(&vault)? { + held.insert(hex::encode(Sha256::digest(std::fs::read(f?.path())?))); + } + } + if let Some(missing) = named.iter().find(|h| !held.contains(*h)) { + return Err(BackupError::Damaged(format!( + "a TIN record the books name isn't in it ({})", + &missing[..12.min(missing.len())] + ))); + } + let key = scratch + .join(crate::root::KEYS_DIR) + .join(crate::root::LEGACY_KEY_FILE); + if !key.is_file() { + return Err(BackupError::Damaged( + "its TIN records are in it, but not the key that opens them".into(), + )); + } + } + // The head the backup records: its commit says "seq N head H". + let recorded = git.run( + "log", + &[os(&git_dir), os("log"), os("-1"), os("--format=%s")], + )?; + if recorded.trim() != format!("seq {} head {}", head.0, head.1) { + return Err(BackupError::Damaged(format!( + "its journal ends at event {}, but the backup records another head", + head.0 + ))); + } + let body = git.run( + "log", + &[os(&git_dir), os("log"), os("-1"), os("--format=%b")], + )?; + let book = body + .lines() + .find_map(|l| l.trim().strip_prefix("book ")) + .map(str::to_string); + Ok(Verified { + head, + blobs, + tree: scratch.to_path_buf(), + book, + }) +} + +/// A restore drill (INVARIANT 3): the backup of the book `book` at +/// `target` cloned into `scratch` (a new folder, removed after), proven +/// whole, and its journal a byte prefix of `live` (the book's journal +/// now). Its head. +pub fn drill( + git: &Git, + target: &Target, + book: &str, + live: &Path, + scratch: &Path, +) -> Result<(u64, String), BackupError> { + if let Target::Folder(folder) = target + && !folder.is_dir() + { + return Err(BackupError::NoFolder); + } + let checked = verify(git, &target.source(book), scratch).and_then(|v| { + match is_prefix(live, &v.tree.join(crate::JOURNAL_FILE)) { + Ok(true) => Ok(v.head), + Ok(false) => Err(BackupError::Damaged( + "its journal isn't this book's history".into(), + )), + Err(e) => Err(e.into()), + } + }); + let _ = std::fs::remove_dir_all(scratch); + checked +} + +/// Whether the journal at `backup` is a byte prefix of the one at `live` +/// (a drill: the backup holds the book's history, nothing else). +pub fn is_prefix(live: &Path, backup: &Path) -> std::io::Result { + use std::io::Read; + let mut a = std::fs::File::open(live)?; + let mut b = std::fs::File::open(backup)?; + let (mut x, mut y) = (vec![0u8; 64 * 1024], vec![0u8; 64 * 1024]); + loop { + let n = b.read(&mut y)?; + if n == 0 { + return Ok(true); + } + let mut got = 0; + while got < n { + let m = a.read(&mut x[got..n])?; + if m == 0 { + return Ok(false); + } + got += m; + } + if x[..n] != y[..n] { + return Ok(false); + } + } +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + use crate::Store; + use ledger_core::{AccountType, Currency, Money, Side}; + + fn usd(s: &str) -> Money { + Money::new(s.parse().unwrap(), Currency::USD) + } + + /// A book with a few entries and a stored file. + fn book(dir: &Path) -> Store { + let mut s = Store::create(dir, Currency::USD).unwrap(); + let cash = s + .add_account("1000", "Cash", AccountType::Asset, None, false) + .unwrap(); + let cap = s + .add_account("3000", "Capital", AccountType::Equity, None, false) + .unwrap(); + let e = s + .create_entry( + chrono::NaiveDate::from_ymd_opt(2026, 3, 1).unwrap(), + "Start", + ) + .unwrap(); + s.add_line(e, cash, Side::Debit, usd("100.00")).unwrap(); + s.add_line(e, cap, Side::Credit, usd("100.00")).unwrap(); + s.post_entry(e).unwrap(); + s.attach_receipt(Some(e), "r.pdf", "application/pdf", b"%PDF-1.4 r", None) + .unwrap(); + s + } + + #[test] + fn a_backup_is_the_book_and_proves_whole() { + let git = Git::find().expect("git is installed"); + let dir = tempfile::tempdir().unwrap(); + let b = dir.path().join("book"); + let mut s = book(&b); + let made = commit(&git, &b, "acme", s.head()) + .unwrap() + .expect("a first backup"); + assert_eq!(made.head, s.head()); + // Nothing new: no commit. + assert_eq!(commit(&git, &b, "acme", s.head()).unwrap(), None); + // To a folder, then proven from it. + let folder = dir.path().join("disk"); + std::fs::create_dir(&folder).unwrap(); + let target = Target::Folder(folder.clone()); + assert_eq!(push(&git, &b, "acme", &target, true).unwrap(), made.commit); + let v = verify( + &git, + folder.join("acme.git").to_str().unwrap(), + &dir.path().join("v1"), + ) + .unwrap(); + assert_eq!(v.head, s.head()); + assert_eq!(v.blobs, 1); + assert!( + is_prefix( + &b.join(crate::JOURNAL_FILE), + &v.tree.join(crate::JOURNAL_FILE) + ) + .unwrap() + ); + // More writes: a new commit, pushed forward. + s.add_account("6000", "Supplies", AccountType::Expense, None, false) + .unwrap(); + let next = commit(&git, &b, "acme", s.head()).unwrap().unwrap(); + assert_ne!(next.commit, made.commit); + push(&git, &b, "acme", &target, true).unwrap(); + let v2 = verify( + &git, + folder.join("acme.git").to_str().unwrap(), + &dir.path().join("v2"), + ) + .unwrap(); + assert_eq!(v2.head, s.head()); + // The older backup is a prefix of the live journal; the live one is + // not a prefix of the older. + assert!( + is_prefix( + &b.join(crate::JOURNAL_FILE), + &v.tree.join(crate::JOURNAL_FILE) + ) + .unwrap() + ); + assert!( + !is_prefix( + &v.tree.join(crate::JOURNAL_FILE), + &b.join(crate::JOURNAL_FILE) + ) + .unwrap() + ); + // The database, its lock and the side channels aren't backed up. + let listed = git + .run( + "ls-tree", + &[ + os(&format!("--git-dir={}", b.join(ARCHIVE_DIR).display())), + os("ls-tree"), + os("--name-only"), + os("HEAD"), + ], + ) + .unwrap(); + let mut names: Vec<&str> = listed.lines().collect(); + names.sort(); + assert_eq!(names, ["blobs", crate::JOURNAL_FILE]); + } + + #[test] + fn a_folder_targets_hooks_never_run() { + use std::os::unix::fs::PermissionsExt; + let git = Git::find().expect("git is installed"); + let dir = tempfile::tempdir().unwrap(); + let b = dir.path().join("book"); + let s = book(&b); + commit(&git, &b, "acme", s.head()).unwrap(); + let folder = dir.path().join("disk"); + let repo = folder.join("acme.git"); + std::fs::create_dir(&folder).unwrap(); + git.run( + "init", + &[os("init"), os("--bare"), os("-q"), repo.as_os_str()], + ) + .unwrap(); + let ran = dir.path().join("ran"); + std::fs::create_dir_all(repo.join("hooks")).unwrap(); + for hook in ["pre-receive", "update", "post-receive", "post-update"] { + let h = repo.join("hooks").join(hook); + std::fs::write( + &h, + format!("#!/bin/sh\necho {hook} >> '{}'\n", ran.display()), + ) + .unwrap(); + std::fs::set_permissions(&h, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + // As a folder, and named as a repository by its path. + push(&git, &b, "acme", &Target::Folder(folder.clone()), true).unwrap(); + push( + &git, + &b, + "acme", + &Target::Remote(repo.display().to_string()), + true, + ) + .unwrap(); + assert!( + !ran.exists(), + "a hook ran: {:?}", + std::fs::read_to_string(&ran) + ); + let v = verify(&git, repo.to_str().unwrap(), &dir.path().join("v")).unwrap(); + assert_eq!(v.head, s.head()); + } + + #[test] + fn a_target_that_moved_on_is_never_overwritten() { + let git = Git::find().expect("git is installed"); + let dir = tempfile::tempdir().unwrap(); + let folder = dir.path().join("disk"); + std::fs::create_dir(&folder).unwrap(); + let target = Target::Folder(folder.clone()); + // Two books of the same id push to one folder: the second has its + // own history, which isn't the first's. + let one = dir.path().join("one"); + let s = book(&one); + commit(&git, &one, "acme", s.head()).unwrap(); + push(&git, &one, "acme", &target, true).unwrap(); + let two = dir.path().join("two"); + let mut s2 = book(&two); + s2.add_account("6000", "Supplies", AccountType::Expense, None, false) + .unwrap(); + commit(&git, &two, "acme", s2.head()).unwrap(); + let e = push(&git, &two, "acme", &target, true).unwrap_err(); + assert!(matches!(e, BackupError::Diverged(_)), "{e}"); + // The folder still holds the first's. + let v = verify( + &git, + folder.join("acme.git").to_str().unwrap(), + &dir.path().join("v"), + ) + .unwrap(); + assert_eq!(v.head, s.head()); + } + + #[test] + fn a_missing_folder_or_a_strange_address_is_said_plainly() { + let git = Git::find().expect("git is installed"); + let dir = tempfile::tempdir().unwrap(); + let b = dir.path().join("book"); + let s = book(&b); + commit(&git, &b, "acme", s.head()).unwrap(); + let e = push( + &git, + &b, + "acme", + &Target::Folder(dir.path().join("unplugged")), + true, + ) + .unwrap_err(); + assert_eq!(e, BackupError::NoFolder); + // Never read as an option, never empty: refused before git runs. + for url in ["--upload-pack=touch /x", "-x", "", " "] { + let e = push(&git, &b, "acme", &Target::Remote(url.into()), true).unwrap_err(); + assert!(matches!(e, BackupError::Damaged(_)), "{url}: {e}"); + } + // Never a command. + assert!( + push( + &git, + &b, + "acme", + &Target::Remote("ext::sh -c 'touch x'".into()), + true + ) + .is_err() + ); + assert!(!Path::new("x").exists()); + // A source that isn't a backup can't be trusted. + let e = verify( + &git, + dir.path().join("nothing").to_str().unwrap(), + &dir.path().join("v"), + ) + .unwrap_err(); + assert!(e.to_string().contains("name the repository itself"), "{e}"); + let e = verify( + &git, + "ssh://nowhere.invalid/books.git", + &dir.path().join("v"), + ) + .unwrap_err(); + assert!(matches!(e, BackupError::Git(..)), "{e}"); + let text = e.to_string(); + assert!(!text.contains(&b.display().to_string()), "{text}"); + } + + #[test] + fn no_ignore_rule_leaves_a_file_out() { + let git = Git::find().expect("git is installed"); + let dir = tempfile::tempdir().unwrap(); + let b = dir.path().join("book"); + let s = book(&b); + for (d, f) in [ + (crate::root::VAULT_DIR, "tin-1.bin"), + (crate::root::KEYS_DIR, "legacy.key"), + ] { + std::fs::create_dir(b.join(d)).unwrap(); + std::fs::write(b.join(d).join(f), b"x").unwrap(); + } + commit(&git, &b, "acme", s.head()).unwrap(); + // Ignore rules where git looks for them, as a template or a hand + // might leave them. + std::fs::create_dir_all(b.join(ARCHIVE_DIR).join("info")).unwrap(); + std::fs::write( + b.join(ARCHIVE_DIR).join("info").join("exclude"), + "*.key\n*.bin\nblobs/\n", + ) + .unwrap(); + std::fs::write(b.join(crate::root::VAULT_DIR).join("tin-2.bin"), b"y").unwrap(); + commit(&git, &b, "acme", s.head()) + .unwrap() + .expect("the new record is a change"); + let listed = git + .run( + "ls-tree", + &[ + os(&format!("--git-dir={}", b.join(ARCHIVE_DIR).display())), + os("ls-tree"), + os("-r"), + os("--name-only"), + os("HEAD"), + ], + ) + .unwrap(); + for want in [ + "keys/legacy.key", + "vault/tin-1.bin", + "vault/tin-2.bin", + crate::JOURNAL_FILE, + ] { + assert!(listed.lines().any(|l| l == want), "{want}: {listed}"); + } + assert!(listed.lines().any(|l| l.starts_with("blobs/")), "{listed}"); + } + + #[test] + fn what_git_said_keeps_whole_lines_and_drops_its_progress() { + let text = b"Counting objects: 100% (3/3), done.\rWriting objects: 50% (1/2)\r\nfatal: '/root/books/acme' is not a repository\n"; + assert_eq!( + said(text, false), + "fatal: '/root/books/acme' is not a repository" + ); + // A line too long to keep whole is left out, never cut. + let long = format!("fatal: {}\nerror: short\n", "x".repeat(2000)); + assert_eq!(said(long.as_bytes(), false), "error: short"); + // A long line last: nothing before it is reached past it. + let last = format!("error: first\nfatal: {}\n", "x".repeat(2000)); + assert_eq!(said(last.as_bytes(), false), ""); + assert_eq!( + said( + b"remote: Enumerating objects: 5, done.\nfatal: gone\n", + false + ), + "fatal: gone" + ); + } + + #[test] + fn a_purged_vault_leaves_the_next_backup() { + let git = Git::find().expect("git is installed"); + let dir = tempfile::tempdir().unwrap(); + let b = dir.path().join("book"); + let s = book(&b); + std::fs::create_dir(b.join(crate::root::VAULT_DIR)).unwrap(); + std::fs::write(b.join(crate::root::VAULT_DIR).join("tin-1.bin"), b"sealed").unwrap(); + commit(&git, &b, "acme", s.head()).unwrap().unwrap(); + std::fs::remove_dir_all(b.join(crate::root::VAULT_DIR)).unwrap(); + commit(&git, &b, "acme", s.head()) + .unwrap() + .expect("the purge is a change"); + let folder = dir.path().join("disk"); + std::fs::create_dir(&folder).unwrap(); + push(&git, &b, "acme", &Target::Folder(folder.clone()), true).unwrap(); + let v = verify( + &git, + folder.join("acme.git").to_str().unwrap(), + &dir.path().join("v"), + ) + .unwrap(); + assert!(!v.tree.join(crate::root::VAULT_DIR).exists()); + } + + #[test] + fn a_lock_left_behind_never_stops_the_next_backup() { + let git = Git::find().expect("git is installed"); + let dir = tempfile::tempdir().unwrap(); + let b = dir.path().join("book"); + let mut s = book(&b); + commit(&git, &b, "acme", s.head()).unwrap().unwrap(); + // A git stopped part way leaves its locks: while one might still be + // at work they stay (the backup waits), once older they go. + let locks = ["index.lock", "HEAD.lock", "refs/heads/main.lock"]; + for lock in locks { + std::fs::write(b.join(ARCHIVE_DIR).join(lock), b"").unwrap(); + } + s.add_account("6000", "Supplies", AccountType::Expense, None, false) + .unwrap(); + assert!(commit(&git, &b, "acme", s.head()).is_err()); + let then = std::time::SystemTime::now() - GIT_TIMEOUT - Duration::from_secs(1); + for lock in locks { + std::fs::File::options() + .write(true) + .open(b.join(ARCHIVE_DIR).join(lock)) + .unwrap() + .set_modified(then) + .unwrap(); + } + let made = commit(&git, &b, "acme", s.head()) + .unwrap() + .expect("backed up"); + assert_eq!(made.head, s.head()); + for lock in locks { + assert!(!b.join(ARCHIVE_DIR).join(lock).exists(), "{lock}"); + } + } + + #[test] + fn a_backup_with_a_changed_stored_file_or_a_busy_scratch_is_refused() { + let git = Git::find().expect("git is installed"); + let dir = tempfile::tempdir().unwrap(); + let b = dir.path().join("book"); + let s = book(&b); + commit(&git, &b, "acme", s.head()).unwrap(); + let folder = dir.path().join("disk"); + std::fs::create_dir(&folder).unwrap(); + push(&git, &b, "acme", &Target::Folder(folder.clone()), true).unwrap(); + let source = folder.join("acme.git").display().to_string(); + // A scratch folder in use is never written into. + let busy = dir.path().join("busy"); + std::fs::create_dir(&busy).unwrap(); + let e = verify(&git, &source, &busy).unwrap_err(); + assert!(matches!(e, BackupError::Damaged(_)), "{e}"); + assert_eq!(std::fs::read_dir(&busy).unwrap().count(), 0); + // One stored file's bytes changed, its name kept. + let work = dir.path().join("work"); + git.run( + "clone", + &[os("clone"), os("-q"), os(&source), work.as_os_str()], + ) + .unwrap(); + let mut blob = None; + for f in walk(&work.join("blobs")) { + blob = Some(f); + } + let blob = blob.expect("a stored file"); + std::fs::write(&blob, b"not the receipt").unwrap(); + let w = format!("--git-dir={}", work.join(".git").display()); + let t = format!("--work-tree={}", work.display()); + git.run( + "commit", + &[ + os(&w), + os(&t), + os("commit"), + os("-qam"), + os(&format!("seq {} head {}", s.head().0, s.head().1)), + ], + ) + .unwrap(); + let bad = dir.path().join("bad.git"); + git.run( + "clone", + &[ + os("clone"), + os("-q"), + os("--bare"), + work.as_os_str(), + bad.as_os_str(), + ], + ) + .unwrap(); + let e = verify(&git, bad.to_str().unwrap(), &dir.path().join("v")).unwrap_err(); + assert!(matches!(e, BackupError::Damaged(_)), "{e}"); + } + + #[test] + fn a_drill_proves_this_books_history_only() { + let git = Git::find().expect("git is installed"); + let dir = tempfile::tempdir().unwrap(); + let folder = dir.path().join("disk"); + std::fs::create_dir(&folder).unwrap(); + let target = Target::Folder(folder.clone()); + let one = dir.path().join("one"); + let s = book(&one); + commit(&git, &one, "acme", s.head()).unwrap(); + push(&git, &one, "acme", &target, true).unwrap(); + let journal = one.join(crate::JOURNAL_FILE); + let scratch = dir.path().join("drill"); + assert_eq!( + drill(&git, &target, "acme", &journal, &scratch).unwrap(), + s.head() + ); + assert!(!scratch.exists(), "the scratch folder is removed"); + // Another book's backup, whole in itself, isn't this book's. + // (Another currency: its journal differs from its first line.) + let two = dir.path().join("two"); + let mut s2 = Store::create(&two, Currency::new("EUR").unwrap()).unwrap(); + s2.add_account("6000", "Supplies", AccountType::Expense, None, false) + .unwrap(); + let e = drill( + &git, + &target, + "acme", + &two.join(crate::JOURNAL_FILE), + &scratch, + ) + .unwrap_err(); + assert!(e.to_string().contains("isn't this book's history"), "{e}"); + assert!(!scratch.exists()); + // A folder that isn't there. + let gone = Target::Folder(dir.path().join("unplugged")); + assert_eq!( + drill(&git, &gone, "acme", &journal, &scratch).unwrap_err(), + BackupError::NoFolder + ); + // Where each target's backup is. + assert_eq!( + target.source("acme"), + folder.join("acme.git").display().to_string() + ); + assert_eq!( + Target::Remote("ssh://h/b".into()).source("acme"), + "ssh://h/b" + ); + } + + #[test] + fn a_backup_holding_a_link_or_a_stranger_is_refused_before_checkout() { + let git = Git::find().expect("git is installed"); + let dir = tempfile::tempdir().unwrap(); + let b = dir.path().join("book"); + let s = book(&b); + commit(&git, &b, "acme", s.head()).unwrap(); + let folder = dir.path().join("disk"); + std::fs::create_dir(&folder).unwrap(); + push(&git, &b, "acme", &Target::Folder(folder.clone()), true).unwrap(); + let subject = format!("seq {} head {}", s.head().0, s.head().1); + let planted = dir.path().join("planted.db"); + type Spoil = fn(&Path, &Path); + let spoils: [(&str, Spoil); 4] = [ + ("db-link", |w, planted| { + std::os::unix::fs::symlink(planted, w.join("wanderbooks.db")).unwrap(); + }), + ("vault-link", |w, _| { + std::os::unix::fs::symlink("/etc", w.join(crate::root::VAULT_DIR)).unwrap(); + }), + ("journal-link", |w, _| { + let j = w.join(crate::JOURNAL_FILE); + let real = w.join("elsewhere.jsonl"); + std::fs::rename(&j, &real).unwrap(); + std::os::unix::fs::symlink(&real, &j).unwrap(); + }), + ("stranger", |w, _| { + std::fs::write(w.join("README"), b"hello").unwrap(); + }), + ]; + for (tag, spoil) in spoils { + let work = dir.path().join(format!("work-{tag}")); + let source = folder.join("acme.git"); + let out = Command::new("git") + .args(["clone", "-q"]) + .arg(&source) + .arg(&work) + .output() + .unwrap(); + assert!(out.status.success(), "{out:?}"); + spoil(&work, &planted); + let out = Command::new("git") + .arg("-C") + .arg(&work) + .args(["-c", "user.name=t", "-c", "user.email=t@t.invalid"]) + .args(["add", "-A", "."]) + .output() + .unwrap(); + assert!(out.status.success(), "{out:?}"); + let out = Command::new("git") + .arg("-C") + .arg(&work) + .args([ + "-c", + "user.name=t", + "-c", + "user.email=t@t.invalid", + "-c", + "commit.gpgsign=false", + ]) + .args(["commit", "-qm", &subject]) + .output() + .unwrap(); + assert!(out.status.success(), "{tag}: {out:?}"); + let bad = dir.path().join(format!("{tag}.git")); + let out = Command::new("git") + .args(["clone", "-q", "--bare"]) + .arg(&work) + .arg(&bad) + .output() + .unwrap(); + assert!(out.status.success(), "{out:?}"); + let scratch = dir.path().join(format!("v-{tag}")); + let e = verify(&git, bad.to_str().unwrap(), &scratch).unwrap_err(); + assert!(matches!(e, BackupError::Damaged(_)), "{tag}: {e}"); + assert!(!planted.exists(), "{tag}: wrote outside its folder"); + assert!( + !scratch.join(crate::JOURNAL_FILE).exists(), + "{tag}: checked out" + ); + } + } + + #[test] + fn local_addresses_are_told_from_networked_ones() { + for (address, local) in [ + ("/srv/books.git", true), + ("file:///srv/books.git", true), + ("acme.git", true), + ("backups/acme.git", true), + ("./x:y", true), + ("ssh://host/books.git", false), + ("https://host/owner/books", false), + ("git@host:owner/books.git", false), + ("host:books.git", false), + ] { + assert_eq!(is_local(address), local, "{address}"); + } + } + + #[test] + fn a_file_git_wont_take_fails_the_backup() { + let git = Git::find().expect("git is installed"); + let dir = tempfile::tempdir().unwrap(); + let b = dir.path().join("book"); + let s = book(&b); + std::fs::create_dir(b.join(crate::root::VAULT_DIR)).unwrap(); + // git leaves out a file named .git without a word. + std::fs::write(b.join(crate::root::VAULT_DIR).join(".git"), b"x").unwrap(); + let e = commit(&git, &b, "acme", s.head()).unwrap_err(); + assert!(matches!(e, BackupError::Damaged(_)), "{e}"); + } + + #[test] + fn no_attributes_rule_changes_a_byte() { + let git = Git::find().expect("git is installed"); + let dir = tempfile::tempdir().unwrap(); + let b = dir.path().join("book"); + let s = book(&b); + std::fs::create_dir(b.join(crate::root::VAULT_DIR)).unwrap(); + let bytes = b"$Id: kept$\r\nline two\r\n"; + std::fs::write(b.join(crate::root::VAULT_DIR).join("tin-1.bin"), bytes).unwrap(); + commit(&git, &b, "acme", s.head()).unwrap(); + // The archive's own rules are taken out before each backup. + let info = b.join(ARCHIVE_DIR).join("info"); + std::fs::create_dir_all(&info).unwrap(); + std::fs::write(info.join("attributes"), "* ident text eol=lf\n").unwrap(); + std::fs::write(b.join(crate::root::VAULT_DIR).join("tin-2.bin"), bytes).unwrap(); + commit(&git, &b, "acme", s.head()).unwrap().unwrap(); + assert!(!info.join("attributes").exists()); + let gd = format!("--git-dir={}", b.join(ARCHIVE_DIR).display()); + for f in ["vault/tin-1.bin", "vault/tin-2.bin"] { + let out = Command::new(&git.exe) + .args([&gd, "cat-file", "-p", &format!("HEAD:{f}")]) + .output() + .unwrap(); + assert_eq!(out.stdout, bytes, "{f}"); + } + // A rules file in the book's folder, or in what is backed up: + // refused. + for at in [ + b.join(".gitattributes"), + b.join(crate::root::VAULT_DIR).join(".gitattributes"), + ] { + std::fs::write(&at, "* ident\n").unwrap(); + let e = commit(&git, &b, "acme", s.head()).unwrap_err(); + assert!(e.to_string().contains(".gitattributes"), "{e}"); + std::fs::remove_file(&at).unwrap(); + } + } + + #[test] + fn the_archives_own_hooks_never_run() { + use std::os::unix::fs::PermissionsExt; + let git = Git::find().expect("git is installed"); + let dir = tempfile::tempdir().unwrap(); + let b = dir.path().join("book"); + let mut s = book(&b); + commit(&git, &b, "acme", s.head()).unwrap(); + let ran = dir.path().join("ran"); + let hooks = b.join(ARCHIVE_DIR).join("hooks"); + std::fs::create_dir_all(&hooks).unwrap(); + for hook in [ + "pre-commit", + "post-commit", + "reference-transaction", + "pre-push", + "post-index-change", + ] { + let h = hooks.join(hook); + std::fs::write( + &h, + format!("#!/bin/sh\necho {hook} >> '{}'\n", ran.display()), + ) + .unwrap(); + std::fs::set_permissions(&h, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + s.add_account("6000", "Supplies", AccountType::Expense, None, false) + .unwrap(); + commit(&git, &b, "acme", s.head()).unwrap().unwrap(); + let folder = dir.path().join("disk"); + std::fs::create_dir(&folder).unwrap(); + push(&git, &b, "acme", &Target::Folder(folder), true).unwrap(); + assert!( + !ran.exists(), + "a hook ran: {:?}", + std::fs::read_to_string(&ran) + ); + } + + /// A git that runs the real one with the owner's own configuration set + /// to everything that could spoil a backup. + fn hostile_git(dir: &Path, real: &Git) -> Git { + use std::os::unix::fs::PermissionsExt; + let home = dir.join("hostile-home"); + let hooks = home.join("hooks"); + std::fs::create_dir_all(&hooks).unwrap(); + let marker = dir.join("marker"); + for hook in [ + "post-commit", + "pre-push", + "post-receive", + "reference-transaction", + ] { + let h = hooks.join(hook); + std::fs::write( + &h, + format!("#!/bin/sh\necho {hook} >> '{}'\n", marker.display()), + ) + .unwrap(); + std::fs::set_permissions(&h, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + std::fs::write(home.join("ignore"), "*.bin\n*.key\nblobs/\n").unwrap(); + std::fs::write(home.join("attributes"), "* text=auto eol=lf ident\n").unwrap(); + std::fs::write( + home.join(".gitconfig"), + format!( + "[core]\n\tautocrlf = true\n\texcludesFile = {h}/ignore\n\tattributesFile = {h}/attributes\n\thooksPath = {h}/hooks\n\tfsmonitor = touch '{m}.fsmonitor'\n\tsymlinks = true\n[commit]\n\tgpgsign = true\n[push]\n\tgpgSign = true\n[gpg]\n\tprogram = false\n[init]\n\tdefaultRefFormat = reftable\n\ttemplateDir = {h}/template\n[protocol \"ext\"]\n\tallow = always\n[log]\n\tshowSignature = true\n", + h = home.display(), + m = marker.display() + ), + ) + .unwrap(); + let wrapper = dir.join("git"); + std::fs::write( + &wrapper, + format!( + "#!/bin/sh\nexport HOME='{}'\nexport XDG_CONFIG_HOME='{}'\nexec '{}' \"$@\"\n", + home.display(), + home.display(), + real.exe.display() + ), + ) + .unwrap(); + std::fs::set_permissions(&wrapper, std::fs::Permissions::from_mode(0o755)).unwrap(); + Git { + exe: wrapper, + quiet: GIT_TIMEOUT, + } + } + + #[test] + fn the_owners_git_configuration_never_spoils_a_backup() { + let real = Git::find().expect("git is installed"); + let dir = tempfile::tempdir().unwrap(); + let git = hostile_git(dir.path(), &real); + let b = dir.path().join("book"); + let s = book(&b); + std::fs::create_dir(b.join(crate::root::VAULT_DIR)).unwrap(); + let crlf = b"$Id: x$\r\nline\r\n"; + std::fs::write(b.join(crate::root::VAULT_DIR).join("tin-1.bin"), crlf).unwrap(); + std::fs::create_dir(b.join(crate::root::KEYS_DIR)).unwrap(); + std::fs::write(b.join(crate::root::KEYS_DIR).join("legacy.key"), b"k").unwrap(); + commit(&git, &b, "acme", s.head()).unwrap().unwrap(); + let folder = dir.path().join("disk"); + std::fs::create_dir(&folder).unwrap(); + push(&git, &b, "acme", &Target::Folder(folder.clone()), true).unwrap(); + let v = verify( + &git, + folder.join("acme.git").to_str().unwrap(), + &dir.path().join("v"), + ) + .unwrap(); + assert_eq!(v.head, s.head()); + assert_eq!(std::fs::read(v.tree.join("vault/tin-1.bin")).unwrap(), crlf); + assert!(v.tree.join("keys/legacy.key").is_file()); + assert!(!dir.path().join("marker").exists(), "a hook ran"); + assert!( + !dir.path().join("marker.fsmonitor").exists(), + "a watcher ran" + ); + assert!( + push( + &git, + &b, + "acme", + &Target::Remote("ext::sh -c 'touch x'".into()), + true + ) + .is_err() + ); + } + + /// A stand-in for git: a script. + fn fake_git(dir: &Path, script: &str, quiet: Duration) -> Git { + use std::os::unix::fs::PermissionsExt; + let exe = dir.join("fake-git"); + std::fs::write(&exe, format!("#!/bin/sh\n{script}\n")).unwrap(); + std::fs::set_permissions(&exe, std::fs::Permissions::from_mode(0o755)).unwrap(); + Git { exe, quiet } + } + + #[test] + fn a_git_is_stopped_only_once_it_falls_quiet_and_with_all_it_started() { + let dir = tempfile::tempdir().unwrap(); + let limit = Duration::from_millis(500); + // Talking all along, for longer than the limit: let run. + let talking = fake_git( + dir.path(), + "for i in 1 2 3 4 5 6 7; do echo $i >&2; sleep 0.2; done; echo done", + limit, + ); + assert_eq!(talking.run("talk", &[]).unwrap(), "done"); + // Quiet: stopped, and what it started too. + let pid = dir.path().join("pid"); + let quiet = fake_git( + dir.path(), + &format!("sleep 30 & echo $! > '{}'; sleep 30", pid.display()), + Duration::from_millis(300), + ); + let started = Instant::now(); + assert_eq!( + quiet.run("wait", &[]).unwrap_err(), + BackupError::TimedOut("wait".into()) + ); + assert!(started.elapsed() < Duration::from_secs(10)); + let child: i32 = std::fs::read_to_string(&pid) + .unwrap() + .trim() + .parse() + .unwrap(); + std::thread::sleep(Duration::from_millis(200)); + // SAFETY: signal 0 only asks whether the process is there. + assert_ne!( + unsafe { libc::kill(child, 0) }, + 0, + "what it started still runs" + ); + // Done, leaving something holding its output: not waited for. + let lingering = fake_git(dir.path(), "(sleep 30 &) ; echo ok", GIT_TIMEOUT); + let started = Instant::now(); + assert_eq!(lingering.run("linger", &[]).unwrap(), "ok"); + assert!(started.elapsed() < Duration::from_secs(5)); + // Failing: what it said, in whole lines. + let failing = fake_git(dir.path(), "echo 'fatal: no' >&2; exit 1", GIT_TIMEOUT); + assert_eq!( + failing.run("fail", &[]).unwrap_err(), + BackupError::Git("fail".into(), "fatal: no".into()) + ); + // Said at length, then failing: the end of it is kept (where the + // error is), from a whole line on. + let chatty = fake_git( + dir.path(), + "i=0; while [ $i -lt 30000 ]; do echo \"remote: line $i of a long story that goes on\" >&2; i=$((i+1)); done; echo 'fatal: the end' >&2; exit 1", + GIT_TIMEOUT, + ); + match chatty.run("chat", &[]).unwrap_err() { + BackupError::Git(_, said) => { + assert!(said.ends_with("fatal: the end"), "{said}"); + assert!( + said.lines() + .all(|l| l.starts_with("remote: line ") || l == "fatal: the end"), + "{said}" + ); + } + other => panic!("{other:?}"), + } + // Where its path holds a quote, a push to this computer's own + // repository is refused (its hooks couldn't be turned off). + let quoted = dir.path().join("it's"); + std::fs::create_dir("ed).unwrap(); + let real = Git::find().expect("git is installed"); + std::os::unix::fs::symlink(&real.exe, quoted.join("git")).unwrap(); + let odd = Git { + exe: quoted.join("git"), + quiet: GIT_TIMEOUT, + }; + let b = dir.path().join("book"); + let s = book(&b); + commit(&real, &b, "acme", s.head()).unwrap(); + let disk = dir.path().join("disk"); + std::fs::create_dir(&disk).unwrap(); + let e = push(&odd, &b, "acme", &Target::Folder(disk), true).unwrap_err(); + assert!(e.to_string().contains("quote"), "{e}"); + // A long answer is kept whole (only what it says of itself is cut). + let long = fake_git( + dir.path(), + "i=0; while [ $i -lt 40000 ]; do echo blobs/ab/cd/0123456789012345678901234567890123456789; i=$((i+1)); done", + GIT_TIMEOUT, + ); + assert_eq!(long.run("list", &[]).unwrap().lines().count(), 40000); + } + + #[test] + fn a_repository_with_settings_of_its_own_is_never_pushed_to_or_read() { + let git = Git::find().expect("git is installed"); + let dir = tempfile::tempdir().unwrap(); + let b = dir.path().join("book"); + let s = book(&b); + commit(&git, &b, "acme", s.head()).unwrap(); + let folder = dir.path().join("disk"); + std::fs::create_dir(&folder).unwrap(); + let target = Target::Folder(folder.clone()); + push(&git, &b, "acme", &target, true).unwrap(); + let repo = folder.join("acme.git"); + let marker = dir.path().join("ran"); + let config = std::fs::read_to_string(repo.join("config")).unwrap(); + // A command to run on a push; then objects borrowed from elsewhere. + std::fs::write( + repo.join("config"), + format!( + "{config}[core]\n\talternateRefsCommand = touch '{}'\n", + marker.display() + ), + ) + .unwrap(); + for e in [ + push(&git, &b, "acme", &target, true).unwrap_err(), + push( + &git, + &b, + "acme", + &Target::Remote(repo.display().to_string()), + true, + ) + .unwrap_err(), + verify(&git, repo.to_str().unwrap(), &dir.path().join("v1")).unwrap_err(), + ] { + assert!(e.to_string().contains("settings of its own"), "{e}"); + } + // Where a copy came from is no setting of its own; a command to + // run on its way in is. + std::fs::write( + repo.join("config"), + format!("{config}[remote \"origin\"]\n\turl = /x\n\tfetch = +refs/heads/*:refs/remotes/origin/*\n[branch \"main\"]\n\tremote = origin\n\tmerge = refs/heads/main\n"), + ) + .unwrap(); + push(&git, &b, "acme", &target, true).unwrap(); + std::fs::write( + repo.join("config"), + format!( + "{config}[remote \"origin\"]\n\tuploadpack = touch '{}'\n", + marker.display() + ), + ) + .unwrap(); + assert!(push(&git, &b, "acme", &target, true).is_err()); + std::fs::write(repo.join("config"), &config).unwrap(); + std::fs::write(repo.join("objects/info/alternates"), "/elsewhere/objects\n").unwrap(); + let e = verify(&git, repo.to_str().unwrap(), &dir.path().join("v2")).unwrap_err(); + assert!(e.to_string().contains("settings of its own"), "{e}"); + assert!(!marker.exists()); + // Plain again: fine. + std::fs::remove_file(repo.join("objects/info/alternates")).unwrap(); + push(&git, &b, "acme", &target, true).unwrap(); + // Something in it that sends git to another repository first: a + // .git in it, or another's files named in it. + let other = dir.path().join("other.git"); + git.run( + "init", + &[os("init"), os("--bare"), os("-q"), other.as_os_str()], + ) + .unwrap(); + for (name, text) in [ + (".git", format!("gitdir: {}\n", other.display())), + ("commondir", format!("{}\n", other.display())), + ("gitdir", format!("{}\n", other.display())), + ] { + std::fs::write(repo.join(name), text).unwrap(); + let e = push(&git, &b, "acme", &target, true).unwrap_err(); + assert!(e.to_string().contains("settings of its own"), "{name}: {e}"); + let e = verify( + &git, + repo.to_str().unwrap(), + &dir.path().join(format!("v-{name}")), + ) + .unwrap_err(); + assert!(e.to_string().contains("settings of its own"), "{name}: {e}"); + std::fs::remove_file(repo.join(name)).unwrap(); + } + // Something beside it git would take in its place: a repository + // named the same with .git after it, or a bundle. + let decoy = folder.join("acme.git.git"); + git.run( + "init", + &[os("init"), os("--bare"), os("-q"), decoy.as_os_str()], + ) + .unwrap(); + let e = push(&git, &b, "acme", &target, true).unwrap_err(); + assert!(e.to_string().contains("settings of its own"), "{e}"); + std::fs::remove_dir_all(&decoy).unwrap(); + std::fs::write(folder.join("acme.git.bundle"), b"x").unwrap(); + let e = verify(&git, repo.to_str().unwrap(), &dir.path().join("v-bundle")).unwrap_err(); + assert!(e.to_string().contains("settings of its own"), "{e}"); + std::fs::remove_file(folder.join("acme.git.bundle")).unwrap(); + // A HEAD and a config but no objects or refs: not a repository git + // stops at (it would look beside it). + let hollow = dir.path().join("hollow.git"); + std::fs::create_dir(&hollow).unwrap(); + std::fs::write(hollow.join("HEAD"), "ref: refs/heads/main\n").unwrap(); + std::fs::write(hollow.join("config"), "[core]\n\tbare = true\n").unwrap(); + let e = push( + &git, + &b, + "acme", + &Target::Remote(hollow.display().to_string()), + true, + ) + .unwrap_err(); + assert!(e.to_string().contains("name the repository itself"), "{e}"); + // Its parts, or itself, a link elsewhere: refused (what is written + // there would land elsewhere). + let elsewhere = dir.path().join("elsewhere-objects"); + std::fs::rename(repo.join("objects"), &elsewhere).unwrap(); + std::os::unix::fs::symlink(&elsewhere, repo.join("objects")).unwrap(); + let e = push(&git, &b, "acme", &target, true).unwrap_err(); + assert!(e.to_string().contains("settings of its own"), "{e}"); + std::fs::remove_file(repo.join("objects")).unwrap(); + std::fs::rename(&elsewhere, repo.join("objects")).unwrap(); + let moved = dir.path().join("moved.git"); + std::fs::rename(&repo, &moved).unwrap(); + std::os::unix::fs::symlink(&moved, &repo).unwrap(); + let e = push(&git, &b, "acme", &target, true).unwrap_err(); + assert!(e.to_string().contains("settings of its own"), "{e}"); + std::fs::remove_file(&repo).unwrap(); + std::fs::rename(&moved, &repo).unwrap(); + push(&git, &b, "acme", &target, true).unwrap(); + // The folder holding it a link (to a disk's mount, say): fine. + let linked = dir.path().join("linked-disk"); + std::os::unix::fs::symlink(&folder, &linked).unwrap(); + push(&git, &b, "acme", &Target::Folder(linked), true).unwrap(); + // Shared, a copy's origin, a newer ref store: all plain. + let shared = dir.path().join("shared.git"); + git.run( + "init", + &[ + os("init"), + os("--bare"), + os("-q"), + os("--shared"), + shared.as_os_str(), + ], + ) + .unwrap(); + let plain = std::fs::read_to_string(shared.join("config")).unwrap(); + std::fs::write( + shared.join("config"), + format!( + "{plain}[remote \"origin\"]\n\tmirror = true\n[extensions]\n\trefStorage = files\n" + ), + ) + .unwrap(); + plain_repository(&git, shared.to_str().unwrap()).unwrap(); + // Not the repository itself (git would look beside it, or in it). + for address in [ + folder.join("acme"), + folder.clone(), + dir.path().join("nothing-here"), + ] { + let e = push( + &git, + &b, + "acme", + &Target::Remote(address.display().to_string()), + true, + ) + .unwrap_err(); + assert!( + e.to_string().contains("name the repository itself"), + "{address:?}: {e}" + ); + } + assert!(!marker.exists()); + } + + #[test] + fn a_backup_folder_gone_from_a_connected_disk_is_never_made_again() { + let git = Git::find().expect("git is installed"); + let dir = tempfile::tempdir().unwrap(); + let b = dir.path().join("book"); + let s = book(&b); + commit(&git, &b, "acme", s.head()).unwrap(); + // A mount point with nothing mounted: there, but not the backup. + let mount = dir.path().join("mnt"); + std::fs::create_dir(&mount).unwrap(); + let target = Target::Folder(mount.clone()); + assert_eq!( + push(&git, &b, "acme", &target, false).unwrap_err(), + BackupError::NoBackupThere + ); + // A folder that isn't there at all: not connected. + let gone = Target::Folder(dir.path().join("gone")); + assert_eq!( + push(&git, &b, "acme", &gone, false).unwrap_err(), + BackupError::NoFolder + ); + assert!(!mount.join("acme.git").exists()); + // Never backed up there yet: made. + push(&git, &b, "acme", &target, true).unwrap(); + assert!(mount.join("acme.git").join("HEAD").is_file()); + } + + #[test] + fn what_git_said_cut_short_ends_at_its_last_whole_line() { + assert_eq!( + said(b"ot/books/acme'\nfatal: one\nfatal: two", true), + "fatal: one\nfatal: two" + ); + assert_eq!(said(b"ot/books/acme' is not", true), ""); + assert_eq!( + said(b"fatal: one\nfatal: two", false), + "fatal: one\nfatal: two" + ); + } + + /// Every file under `dir`. + fn walk(dir: &Path) -> Vec { + let mut out = Vec::new(); + for e in std::fs::read_dir(dir).unwrap() { + let p = e.unwrap().path(); + if p.is_dir() { + out.extend(walk(&p)); + } else { + out.push(p); + } + } + out + } +} diff --git a/crates/ledger-store/src/lib.rs b/crates/ledger-store/src/lib.rs index 5e1806a..bb95dd2 100644 --- a/crates/ledger-store/src/lib.rs +++ b/crates/ledger-store/src/lib.rs @@ -20,6 +20,7 @@ pub mod access; pub mod auth; +pub mod backup; pub mod blob; pub mod chain; pub mod error; diff --git a/crates/ledger-store/src/root.rs b/crates/ledger-store/src/root.rs index 17a82d2..568a5cc 100644 --- a/crates/ledger-store/src/root.rs +++ b/crates/ledger-store/src/root.rs @@ -32,6 +32,10 @@ //! disk, and its head equal to the source's. //! 6. No error names the root or a path inside it: nothing an agent is //! shown names the books' folder. +//! 7. Restoring a backup (plan 3.7) is proven before use, as an adoption +//! is (INVARIANT 5), and its head must be the one the preview read; +//! it never overwrites: books it replaces are moved whole into +//! `legacy/`, only when asked, and only while nothing holds them. //! ``` use crate::error::StoreError; @@ -70,6 +74,12 @@ pub const LEGACY_KEY_FILE: &str = "legacy.key"; const BOOKS_LOCK: &str = "books.lock"; /// Staging folders under `books/` start so; none is a valid book id. const STAGING_PREFIX: &str = ".adopt-"; +/// A backup being restored, in `books/` (never a book: ids don't start +/// with a dot). +const RESTORE_PREFIX: &str = ".restore-"; +/// Left in a restored book's `ops/`: the id of the books its backup held +/// (empty when it named none), for the server to note when it opens them. +pub const RESTORED_FROM: &str = "restored-from"; /// How the root is named in messages (INVARIANT 6). pub const THE_ROOT: &str = "the books' private folder"; @@ -802,7 +812,276 @@ impl Root { } } +/// What restoring a backup would bring in ([`Root::restore_preview`]). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RestorePreview { + /// The backup's head, `(seq, hash)`, proven. + pub head: (u64, String), + /// Stored files in it, each re-hashed. + pub blobs: u64, + /// Vault records in it. + pub vault_files: u64, + /// Whether books of the id asked for exist now. + pub exists: bool, + /// The id of the books it holds, as they were backed up. + pub origin: Option, +} + +/// What [`Root::restore`] did. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RestoreReport { + /// The restored book's id. + pub id: BookId, + /// Its head, `(seq, hash)`: the backup's. + pub head: (u64, String), + /// Stored files the books name, re-hashed through the store. + pub blobs_verified: u64, + /// Vault records restored. + pub vault_files: u64, + /// The `legacy/` folder the books it replaced were moved to. + pub replaced: Option, + /// The id of the books it held, as they were backed up. + pub origin: Option, +} + impl Root { + /// Clone the backup at `source` (a git repository's address, or its + /// folder: `.git` in a backup folder) into a scratch folder in `books/` and prove it + /// whole (backup.rs INVARIANT 3); the caller holds the books lock and + /// removes the folder. + fn fetch_backup( + &self, + git: &crate::backup::Git, + source: &str, + books: &Path, + id: &BookId, + ) -> Result<(PathBuf, crate::backup::Verified), RootError> { + // Anything left from one that never finished goes first: none is + // in progress (each holds the books lock). + for entry in std::fs::read_dir(books)? { + let entry = entry?; + if entry + .file_name() + .to_str() + .is_some_and(|n| n.starts_with(RESTORE_PREFIX)) + { + std::fs::remove_dir_all(entry.path())?; + } + } + let scratch = books.join(format!("{RESTORE_PREFIX}{id}-{}", std::process::id())); + match crate::backup::verify(git, source, &scratch) { + Ok(v) => Ok((scratch, v)), + Err(e) => { + let _ = std::fs::remove_dir_all(&scratch); + Err(RootError::Adopt(format!( + "that backup can't be restored: {e}" + ))) + } + } + } + + /// What restoring the backup at `source` as books `id` would bring in, + /// proven as [`Self::restore`] proves it, adding nothing to the root. + pub fn restore_preview( + &self, + git: &crate::backup::Git, + source: &str, + id: &BookId, + ) -> Result { + (|| { + let _lock = self.books_lock()?; + let (books, _) = self.private_subdir(BOOKS_DIR, false)?; + let (scratch, v) = self.fetch_backup(git, source, &books, id)?; + let vault = scratch.join(VAULT_DIR); + let vault_files = if vault.is_dir() { + std::fs::read_dir(&vault) + .map(|d| d.count() as u64) + .unwrap_or(0) + } else { + 0 + }; + let _ = std::fs::remove_dir_all(&scratch); + Ok(RestorePreview { + head: v.head, + blobs: v.blobs, + vault_files, + exists: std::fs::symlink_metadata(books.join(id.as_str())).is_ok(), + origin: v.book, + }) + })() + .map_err(|e| self.hide(e)) + } + + /// Restore the backup at `source` as books `id` (INVARIANT 7): proven + /// whole, its head `expect_head` (as the preview read it), built as an + /// adoption builds (every stored file re-hashed, opened, verified, + /// synced) with its backup history (so its next backups carry on from + /// it), and only then renamed into place. Books `id` that exist are + /// refused, unless `replace`: then they are moved into `legacy/` + /// first, kept whole; refused while anything holds them open. + pub fn restore( + &self, + git: &crate::backup::Git, + source: &str, + id: &BookId, + expect_head: &(u64, String), + replace: bool, + ) -> Result { + self.restore_inner(git, source, id, expect_head, replace) + .map_err(|e| match e { + // The copy's checks are the adoption's: said as a restore. + RootError::Adopt(text) => { + RootError::Adopt(text.replace("nothing was adopted", "nothing was restored")) + } + other => other, + }) + .map_err(|e| self.hide(e)) + } + + fn restore_inner( + &self, + git: &crate::backup::Git, + source: &str, + id: &BookId, + expect_head: &(u64, String), + replace: bool, + ) -> Result { + let _lock = self.books_lock()?; + let (books, _) = self.private_subdir(BOOKS_DIR, false)?; + let dest = books.join(id.as_str()); + let exists = std::fs::symlink_metadata(&dest).is_ok(); + if exists && !replace { + return Err(RootError::Exists(id.to_string())); + } + if !exists && replace { + return Err(RootError::Adopt(format!( + "there are no books called {:?} to replace: nothing was restored", + id.as_str() + ))); + } + // Books being replaced are moved only while nothing holds them. + let _held = if exists { + Some(lock_source(&dest).map_err(|_| { + RootError::Adopt(format!( + "the books called {:?} are open: close them first; nothing was restored", + id.as_str() + )) + })?) + } else { + None + }; + let (scratch, v) = self.fetch_backup(git, source, &books, id)?; + let staging = books.join(format!("{RESTORE_PREFIX}{id}-{}-book", std::process::id())); + let built = (|| -> Result { + if v.head != *expect_head { + return Err(RootError::Adopt(format!( + "that backup isn't the one previewed (it is now at event {}): nothing was restored; preview it again", + v.head.0 + ))); + } + let (raw, lines) = read_source_journal(&scratch)?; + if head_of(&lines) != v.head { + return Err(RootError::Adopt( + "the backup's journal changed while it was read: nothing was restored".into(), + )); + } + make_private_dir(&staging)?; + write_private_file(&staging.join(JOURNAL_FILE), &raw)?; + let blob_dir = crate::store::BLOBS_DIR; + let copied = copy_blobs(&scratch.join(blob_dir), &staging.join(blob_dir))?; + let vault_files = if scratch.join(VAULT_DIR).is_dir() { + copy_flat_private(&scratch.join(VAULT_DIR), &staging.join(VAULT_DIR))? + } else { + 0 + }; + if scratch.join(KEYS_DIR).is_dir() { + copy_flat_private(&scratch.join(KEYS_DIR), &staging.join(KEYS_DIR))?; + } + let access = scratch.join(crate::access::ACCESS_FILE); + if access.is_file() { + write_private_file( + &staging.join(crate::access::ACCESS_FILE), + &std::fs::read(&access)?, + )?; + } + // Its backup history: the next backup carries on from it. + std::fs::rename( + scratch.join(".git"), + staging.join(crate::backup::ARCHIVE_DIR), + )?; + // Whose books they were, placed with them. + crate::ops::prepare(&staging)?; + write_private_file( + &crate::ops::ops_dir(&staging).join(RESTORED_FROM), + v.book.as_deref().unwrap_or("").as_bytes(), + )?; + let mut store = Store::open(&staging)?; + if store.head() != v.head { + return Err(RootError::Adopt(format!( + "the copy's head {:?} isn't the backup's {:?}: nothing was restored", + store.head(), + v.head + ))); + } + let named: BTreeSet = store.stored_blob_shas()?.into_iter().collect(); + if !named.is_subset(&copied.shas) { + return Err(RootError::Adopt( + "the backup lacks stored files its books name: nothing was restored".into(), + )); + } + store.verify_and_heal()?; + let blobs_verified = store.verify_blobs()?; + let head = store.head(); + drop(store); + sync_dir(&staging)?; + Ok(RestoreReport { + id: id.clone(), + head, + blobs_verified, + vault_files, + replaced: None, + origin: v.book.clone(), + }) + })(); + let _ = std::fs::remove_dir_all(&scratch); + let placed = built.and_then(|mut report| { + if exists { + let (legacy, _) = self.private_subdir(LEGACY_DIR, false)?; + let stamp = chrono::Utc::now().format("%Y%m%d-%H%M%S"); + let mut name = format!("{id}-{stamp}"); + let mut n = 1; + while std::fs::symlink_metadata(legacy.join(&name)).is_ok() { + n += 1; + name = format!("{id}-{stamp}-{n}"); + } + std::fs::rename(&dest, legacy.join(&name))?; + if let Err(e) = std::fs::rename(&staging, &dest) { + // The books it would replace go back where they were. + if std::fs::rename(legacy.join(&name), &dest).is_err() { + return Err(RootError::Adopt(format!( + "the backup couldn't be put in place ({}), and the books it would \ + replace are in the legacy folder as {name}: nothing was restored", + e.kind() + ))); + } + return Err(e.into()); + } + // In place: a folder that won't sync now is said no more + // than any later write's would be. + let _ = sync_dir(&legacy); + report.replaced = Some(name); + } else { + std::fs::rename(&staging, &dest)?; + } + let _ = sync_dir(&books); + Ok(report) + }); + if placed.is_err() { + let _ = std::fs::remove_dir_all(&staging); + } + placed + } + /// The source of an adoption, made absolute: books outside the root /// (and not holding it) with a journal. fn adoptable(&self, source: &Path) -> Result { diff --git a/crates/ledger-store/tests/root.rs b/crates/ledger-store/tests/root.rs index 35a7ec7..888110b 100644 --- a/crates/ledger-store/tests/root.rs +++ b/crates/ledger-store/tests/root.rs @@ -1284,3 +1284,67 @@ fn books_that_changed_since_the_preview_are_not_adopted() { .unwrap(); assert_eq!(report.source_head, now); } + +/// A backup of `store`'s books (in `dir`) in a folder target under +/// `home`: its address. +fn backed_up(home: &Path, dir: &Path, store: &Store) -> Option { + use ledger_store::backup::{self, Git, Target}; + let git = Git::find().ok()?; + backup::commit(&git, dir, "acme", store.head()).unwrap(); + let folder = home.join("disk"); + std::fs::create_dir_all(&folder).unwrap(); + let target = Target::Folder(folder); + backup::push(&git, dir, "acme", &target, true).unwrap(); + Some(target.source("acme")) +} + +#[test] +fn a_restore_never_moves_books_something_holds() { + use ledger_store::backup::Git; + let home = tmp("restore-held"); + let (root, _) = Root::setup(home.path(), None, None).unwrap(); + let acme = id("acme"); + let held = root.create_book(&acme, Currency::USD).unwrap(); + let dir = root.book_dir(&acme); + let source = backed_up(home.path(), &dir, &held).expect("git is installed"); + let git = Git::find().unwrap(); + let preview = root.restore_preview(&git, &source, &acme).unwrap(); + assert!(preview.exists); + // Open (its writer lock held): refused, nothing moved. + let e = root + .restore(&git, &source, &acme, &preview.head, true) + .unwrap_err(); + assert!(e.to_string().contains("are open"), "{e}"); + assert!(dir.join(JOURNAL_FILE).is_file()); + assert!( + !root.path().join("legacy").exists() + || std::fs::read_dir(root.path().join("legacy")) + .unwrap() + .count() + == 0 + ); + // Not over books without --replace; nothing to replace that isn't. + let e = root + .restore(&git, &source, &acme, &preview.head, false) + .unwrap_err(); + assert!(matches!(e, RootError::Exists(_)), "{e}"); + let e = root + .restore(&git, &source, &id("nobody"), &preview.head, true) + .unwrap_err(); + assert!(e.to_string().contains("no books called"), "{e}"); + // Let go: replaced, the old kept whole. + drop(held); + let r = root + .restore(&git, &source, &acme, &preview.head, true) + .unwrap(); + let kept = root.path().join("legacy").join(r.replaced.unwrap()); + assert!(kept.join(JOURNAL_FILE).is_file()); + assert_eq!(r.head, preview.head); + // Nothing of the restore is left behind in books/. + let left: Vec<_> = std::fs::read_dir(root.path().join("books")) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .filter(|n| n.starts_with('.')) + .collect(); + assert!(left.is_empty(), "{left:?}"); +} diff --git a/docs/mcp-interface.md b/docs/mcp-interface.md index 0d0ece5..9a65426 100644 --- a/docs/mcp-interface.md +++ b/docs/mcp-interface.md @@ -352,7 +352,8 @@ seals a new vault record, every time, so none carries it; `idempotent_tools_change_nothing_when_repeated` calls each candidate twice to check). Every tool's `openWorldHint` is false: the books are a closed world, and no tool -reaches beyond them. `books.export_cpa_package`, `books.vendor_1099` +reaches beyond them, but `books.back_up_now`, which pushes the books to +the owner's backups (true). `books.export_cpa_package`, `books.vendor_1099` and `books.get_w9` keep a file for the agent (an export) and record the ones with TINs, so they are not read-only. No tool carries `requiresUserInteraction` (owner decision @@ -584,6 +585,79 @@ through the policy. every stored file. The server runs the same check, and heals the same way, every time it starts. +## Tools — backups + +Books served from the data root back themselves up (plan 3.7, D-30, +D-31): each keeps `archive.git`, a bare repository whose commits are its +journal, stored files, vault, keys and access log as they were (one +commit per backup, made under the store's lock, its message +`seq N head H` and, below, `book `), pushed fast-forward only to the targets the settings +name: `backup.git_remote` (the owner's private repository) and +`backup.folder` (a repository `.git` in a folder on another disk). +Not encrypted: the destinations keep it private. Backups run a little +after the books are last written (`--backup-quiet`, 30 s), when they +open, before they close, and on request; a failed one is retried after +15 minutes. Each target is drilled weekly: cloned, its chain replayed, +every stored file re-hashed, every TIN record the books name there with +its key, its head the one its commit records, its journal a byte prefix +of the live one (a failed drill is tried again at the server's next +hourly look, once 15 minutes have passed). A backup's tree is checked +before any of it is written out: plain files under the backed-up paths +only, never a link or a submodule. git runs with no hooks (in a +repository on this computer it pushes to as well), no ignore or +attributes rules of anyone's (a `.gitattributes` in a book's folder +refuses the backup) and no command-running transports; a repository on +this computer is used only if plain (a bare repository at that very +path, no settings of its own but where a copy came from, nothing sending +git elsewhere, no link where it or its parts are), and a backup folder that took backups and is gone is +reported as a disk not connected, never made again (restored books count +the backups of the books they were). Books restored beside the books they +were (under another id) don't push to those books' repository, which +would stop theirs: `books.backup` says it is paused for them until +another is named. A damaged backup record is set aside and begun again; +it never keeps the books shut. One book's drill holds the others' +backups until it ends (a backup at close waits at most a minute for +it). A target +whose push failed is left out of the backups after writes for 15 +minutes, then tried again, written to or not; the other targets don't +wait for it. Every +commit, push and drill is a record in the book's `ops/backup.jsonl` (a +chain log; a backup never moves the books' head), each target's records +keyed by the target itself. Setting `backup.*` (books.set_setting, or +books.import_settings with a `[backup]` table) needs books:admin, as +backing up does: a backup is the whole book, its vault and key included. +Over stdio neither tool is offered. + +- `books.backup {}` (books:read) → `{ head_seq, targets: [{ target: + "repository" | "folder", at, last_ok_at, through_seq, stale, + last_error, last_drill_at, drill_error, paused }], note }`. `paused`: + not backed up there, these books being restored as a copy of others + (its `last_error` says so). `stale`: the target + lacks the head, and the first try to back up what it lacks (since its + last good backup, or ever) is more than a day old. `last_error`: its + last push's, or a later failed commit's. `note` + says what to set when no target is. `books.start` adds a next step when + none is set up, or one is stale, failing, paused or failed its drill. +- `books.back_up_now { new_disk? }` (books:admin) → the same, after + backing up to every target now. `new_disk`: the backup folder's disk is + new, so a folder that took these books' backups and holds none now is + begun again (otherwise it is a disk not connected). The one tool whose + `openWorldHint` is true. A target paused for books restored as a copy + of others shows `paused` (and is no failure to retry). + +Restoring is the owner's, from a terminal: `wanderbooks books restore +BACKUP [--id ID] [--replace] [--yes]` (the admin socket's `restore`, +previewed then confirmed against the previewed head; without `--id`, the +books keep the id of the books the backup holds, which the preview names, +else the repository's name; under another id they are a copy, and don't +push to those books' repository). The backup is +cloned into the root, proven whole as a drill proves it (but for the +live-journal check: there may be no live books), and built as an +adoption builds; it arrives as new books, or replaces books (`--replace`, +which are let go and moved whole into `legacy/-