P2 slice 7: backups and restore (D-30, D-31)

The server backs each book up itself: a per-book git archive, pushed
fast-forward only to a git remote and to a folder, when the books open,
after a quiet spell, before they close and on request; every commit,
push and drill is a record in ops/backup.jsonl. Drills clone each
target and verify the chain, the blobs, the vault and the head, weekly
and after a failure. `wanderbooks books restore` previews, then
restores beside or (with --replace) over, never moving books something
holds. The archive refuses anything but a plain bare repository and
pins git against the owner's configuration. books.backup and
books.back_up_now for the agent.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01T3gXMVt7XJPi6ens1or9Dj
This commit is contained in:
swaitsandClaude Opus 5.5 committed 2026-10-20 11:14:21 -06:00
1 parent 5bc0633cdd
commit 019d8d76a3
29 files changed
+6443 -136

No files matched your search

+101 -6
View File
@@ -1,6 +1,6 @@
//! The admin socket's messages (`run/admin.sock`, plan 3.1, 3.3, 3.9): how
//! the command line asks the books server to list, create and adopt books,
//! and to make, list and revoke agents' tokens.
//! the command line asks the books server to list, create, adopt and
//! restore books, and to make, list and revoke agents' tokens.
//!
//! ```text
//! GOAL: the command line makes and adopts books without ever touching
@@ -16,9 +16,10 @@
//! that says to restart.
//! 3. No answer names the data root or a path in it, nor echoes a path
//! the caller gave (which could be the root under another name).
//! 4. Adopting is two steps: without `confirmed` the answer is a
//! preview, and nothing is written; `confirmed: true` adopts, and
//! only books still at the head the preview read (`expect_head`).
//! 4. Adopting and restoring are two steps: without `confirmed` the
//! answer is a preview, and nothing is written; `confirmed: true`
//! adopts (restores), and only books (a backup) still at the head the
//! preview read (`expect_head`).
//! 5. Nothing in a settings file is dropped silently: entries the books
//! don't know are refused unless `skip_unknown`, and its notes and
//! comments are in the preview.
@@ -97,6 +98,24 @@ pub enum Request {
#[serde(default)]
skip_unknown: bool,
},
/// Restore a backup (plan 3.7): as new books, or replacing books
/// (which are moved into `legacy/`, kept).
Restore {
/// The books' id.
id: String,
/// The backup: a git repository's address, or its folder
/// (`<book>.git` in the backup folder).
source: String,
/// Replace the books called `id`.
#[serde(default)]
replace: bool,
/// Restore (true), or only preview it.
#[serde(default)]
confirmed: bool,
/// The head the preview read: restored only if still so.
#[serde(default)]
expect_head: Option<(u64, String)>,
},
/// Make a token for an agent (plan 3.3).
TokenCreate {
/// The books it opens.
@@ -237,6 +256,44 @@ pub struct Adopted {
pub problem: Option<String>,
}
/// What a restore would bring in.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct RestorePreview {
/// The backup's head, proven.
pub head: (u64, String),
/// Stored files in it, each re-hashed.
pub blobs: u64,
/// TIN vault records in it.
pub vault_files: u64,
/// Whether books of that id exist now.
pub exists: bool,
/// The id of the books it holds, as they were backed up.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub origin: Option<String>,
}
/// What a restore did.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Restored {
/// The books' id.
pub id: String,
/// Their head: the backup's.
pub head: (u64, String),
/// Stored files the books name, re-hashed through the store.
pub blobs_verified: u64,
/// TIN vault records restored.
pub vault_files: u64,
/// The `legacy/` folder the replaced books were moved to.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub replaced: Option<String>,
/// The id of the books it held, as they were backed up.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub origin: Option<String>,
/// Why the restored books didn't open, if they didn't (they are kept).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub problem: Option<String>,
}
/// The server's answer.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case", deny_unknown_fields)]
@@ -255,6 +312,10 @@ pub enum Response {
Preview(AdoptPreview),
/// Books adopted and open.
Adopted(Adopted),
/// What a restore would bring in.
RestorePreview(RestorePreview),
/// A backup restored, and open.
Restored(Restored),
/// A token made: its text, shown this once (INVARIANT 6).
TokenCreated {
/// The token.
@@ -307,7 +368,7 @@ pub fn call_within(
// A list changes nothing: its only question is whether it answered.
let look = match request {
Request::List | Request::TokenList => None,
Request::Create { .. } | Request::Adopt { .. } => {
Request::Create { .. } | Request::Adopt { .. } | Request::Restore { .. } => {
Some("`wanderbooks books list` shows the books there are")
}
Request::TokenCreate { .. } | Request::TokenRevoke { .. } => {
@@ -417,6 +478,24 @@ mod tests {
}
}
#[test]
fn a_restore_request_is_a_preview_unless_confirmed() {
let r: Envelope = serde_json::from_value(
json!({"proto": 1, "op": "restore", "id": "acme", "source": "/backups/acme.git"}),
)
.unwrap();
assert_eq!(
r.request,
Request::Restore {
id: "acme".into(),
source: "/backups/acme.git".into(),
replace: false,
confirmed: false,
expect_head: None,
}
);
}
#[test]
fn unknown_operations_and_fields_are_refused() {
for bad in [
@@ -455,6 +534,22 @@ mod tests {
text: TokenText("wb_00".into()),
},
Response::Tokens(vec![]),
Response::RestorePreview(RestorePreview {
head: (3, "ab".into()),
blobs: 2,
vault_files: 1,
exists: true,
origin: Some("acme".into()),
}),
Response::Restored(Restored {
id: "acme".into(),
head: (3, "ab".into()),
blobs_verified: 2,
vault_files: 1,
replaced: Some("acme-20261019-120000".into()),
origin: Some("acme".into()),
problem: None,
}),
] {
let text = serde_json::to_string(&r).unwrap();
assert_eq!(serde_json::from_str::<Response>(&text).unwrap(), r);
+994
View File
@@ -0,0 +1,994 @@
//! The server's backups of a book (plan 3.7, D-30, D-31): made a while
//! after the books are written (when the writing stops), when they open,
//! before they close, and when asked; each pushed to the targets the
//! book's settings name; each recorded in the book's `ops/backup.jsonl`;
//! each target's backup drilled weekly.
//!
//! ```text
//! GOAL: every write reaches the owner's backups soon after it is made,
//! and the owner (and agent) can see that it did, or why not.
//! SOURCE: plan 3.7 (triggers, "ops/backup.jsonl", books.backup, Needs
//! attention for a target more than 24 h behind, drills), D-30 (a
//! git repository and a folder), D-31 (plain git).
//! INVARIANTS:
//! 1. A backup is committed under the store's lock (ledger_store::backup
//! INVARIANT 1; its stored files staged before, outside it) and
//! pushed outside it: the books are never held while the network is
//! slow. One run or drill at a time.
//! 2. Every commit, push and drill is a record of `ops/backup.jsonl` (a
//! chain log, not the journal: a backup never moves the books' head),
//! each target's outcome a record of its own, keyed by the target
//! itself (a new folder or address starts with no standing).
//! 3. A target is stale when it lacks the books' head and the first try
//! to back up what it lacks (the first commit or failure since its
//! last good push, or since its first try if it never had one, or
//! since the failed commits that kept it from any) is more than
//! STALE_AFTER old; its last error is its last push's, or a
//! later failed commit's.
//! 4. Nothing here names a path of the data root to an agent.
//! 5. Every target that holds a backup is drilled (ledger_store::backup::
//! drill: cloned, proven whole, its journal the live one's beginning)
//! once DRILL_EVERY has passed since its last good drill, or
//! RETRY_AFTER since a failed one (at the server's next look for
//! drills); a failed drill is said in the standing until one passes.
//! 6. A target whose push failed lately (RETRY_AFTER) is left out of a
//! backup after a write, and only that one; a folder target's
//! repository is made only while the target never took a backup of
//! these books (one gone since is a disk not connected), restored
//! books counting the backups of the books they were.
//! 7. Books restored beside the books they were (under another id) never
//! push to the repository those books back up to (it would stop
//! theirs): it is paused for them, and said, until another is named.
//! 8. A damaged backup record never keeps the books shut: it is set
//! aside and a new one begun.
//! ```
use crate::server::SharedStore;
use ledger_store::backup::{self, BackupError, Git, Target};
use ledger_store::chain::ChainLog;
use serde::Serialize;
use serde_json::{Map, Value, json};
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
/// The book's backup record, in its `ops/`.
pub const LOG_FILE: &str = "backup.jsonl";
/// The record's genesis text.
const GENESIS: &str = "wanderbooks-backup-v1";
/// How long a target may lack the books' head before it is stale.
pub const STALE_AFTER: chrono::Duration = chrono::Duration::hours(24);
/// How often each target's backup is drilled (plan 3.7: weekly).
pub const DRILL_EVERY: chrono::Duration = chrono::Duration::days(7);
/// How long after a failed run (or drill) the next is tried.
pub const RETRY_AFTER: std::time::Duration = std::time::Duration::from_secs(15 * 60);
/// The most of an error kept.
const ERROR_MAX: usize = 500;
/// A book's backups.
pub struct Backups {
book: String,
dir: PathBuf,
store: Arc<SharedStore>,
log: Mutex<ChainLog>,
/// One run at a time.
running: tokio::sync::Mutex<()>,
}
/// One target's standing (INVARIANT 3).
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
pub struct TargetStatus {
/// "repository" or "folder".
pub target: String,
/// Where (the repository's address, or "the backup folder").
pub at: String,
/// When it last took a push, if ever (UTC).
pub last_ok_at: Option<String>,
/// The journal seq it holds, if any.
pub through_seq: Option<u64>,
/// Whether it is stale (INVARIANT 3).
pub stale: bool,
/// Why its last backup failed, if it did.
pub last_error: Option<String>,
/// When its backup was last drilled (UTC), if ever.
pub last_drill_at: Option<String>,
/// Why its last drill failed, if it did.
pub drill_error: Option<String>,
/// Paused for books restored as a copy of others (INVARIANT 7).
pub paused: bool,
}
/// The books' backup standing.
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
pub struct Status {
/// The books' head now.
pub head_seq: u64,
/// Each target the settings name.
pub targets: Vec<TargetStatus>,
/// None set up: what to do.
pub note: Option<String>,
}
/// A target the settings name.
#[derive(Debug, Clone)]
struct Named {
target: Target,
/// Where, as an agent is told it.
at: String,
/// The target itself, as its records name it (INVARIANT 2): a digest
/// of where it is (never a path an agent is shown).
key: String,
}
/// The targets the book's settings name (D-30).
fn targets(store: &ledger_store::Store, book: &str) -> Vec<Named> {
use sha2::{Digest, Sha256};
let s = store.ledger().settings();
let mut out = Vec::new();
if let Some(url) = s.backup_git_remote() {
out.push((Target::Remote(url.to_string()), url.to_string()));
}
if let Some(folder) = s.backup_folder() {
out.push((
Target::Folder(PathBuf::from(folder)),
"the backup folder".to_string(),
));
}
out.into_iter()
.map(|(target, at)| Named {
key: Sha256::digest(target.source(book).as_bytes())[..8]
.iter()
.map(|b| format!("{b:02x}"))
.collect(),
target,
at,
})
.collect()
}
impl Backups {
/// The backups of the book `book`, kept in `dir`, served from `store`.
pub fn open(book: &str, dir: &Path, store: Arc<SharedStore>) -> Result<Self, String> {
ledger_store::ops::prepare(dir)
.map_err(|e| format!("its backup record can't be kept: {e}"))?;
let path = ledger_store::ops::ops_dir(dir).join(LOG_FILE);
let (log, carried) = match ChainLog::open(&path, GENESIS) {
Ok((log, _)) => (log, Vec::new()),
// INVARIANT 8: set aside, begun again, what it knew of whose
// backups the targets hold carried into the new one.
Err(ledger_store::chain::ChainError::Damaged { .. }) => {
let known = known_before_damage(&path);
let aside = path.with_extension(format!(
"jsonl.damaged-{}",
chrono::Utc::now().format("%Y%m%d-%H%M%S")
));
eprintln!(
"wanderbooks server: a backup record was damaged: set aside, begun again"
);
std::fs::rename(&path, &aside)
.map_err(|e| format!("its backup record can't be set aside: {}", e.kind()))?;
let log = ChainLog::open(&path, GENESIS)
.map_err(|e| format!("its backup record can't be used: {e}"))?
.0;
(log, known)
}
Err(e) => return Err(format!("its backup record can't be used: {e}")),
};
let me = Self {
book: book.to_string(),
dir: dir.to_path_buf(),
store,
log: Mutex::new(log),
running: tokio::sync::Mutex::new(()),
};
for fields in carried {
me.record(fields);
}
// Books just restored: whose they were (INVARIANTS 6, 7).
let marker = ledger_store::ops::ops_dir(dir).join(ledger_store::root::RESTORED_FROM);
if let Ok(origin) = std::fs::read_to_string(&marker) {
let origin = Some(origin.trim()).filter(|o| !o.is_empty());
let named = me
.store
.try_with_store(|s| targets(s, book))
.ok_or("its backups can't be read yet: the books are busy")?;
me.restored(origin, &named);
std::fs::remove_file(&marker)
.map_err(|e| format!("its backup record can't be kept: {}", e.kind()))?;
}
Ok(me)
}
fn record(&self, fields: Value) {
let Value::Object(mut map) = fields else {
return;
};
// INVARIANT 4: a git or store message may quote a path in the root.
if let Some(Value::String(e)) = map.get_mut("error") {
*e = hide(e, &self.dir);
}
let mut log = self.log.lock().unwrap_or_else(|e| e.into_inner());
if let Err(e) = log.append(map) {
eprintln!("wanderbooks server: the books' backup record couldn't be written: {e}");
}
}
fn records(&self) -> Vec<Map<String, Value>> {
ledger_store::chain::read(
&ledger_store::ops::ops_dir(&self.dir).join(LOG_FILE),
GENESIS,
)
.map(|r| r.records)
.unwrap_or_default()
}
/// The targets the settings name now.
async fn named(&self) -> Vec<Named> {
let (store, book) = (self.store.clone(), self.book.clone());
tokio::task::spawn_blocking(move || store.with_store(|s| targets(s, &book)).ok())
.await
.ok()
.flatten()
.unwrap_or_default()
}
/// Note that these books were just restored from a backup of the books
/// `origin` (INVARIANTS 6, 7), `named` the targets they name: the same
/// books (their targets took their backups), or others' (the
/// repository those back up to is paused for these).
fn restored(&self, origin: Option<&str>, named: &[Named]) {
let same = origin == Some(self.book.as_str());
let keys = |remote: bool| -> Vec<String> {
named
.iter()
.filter(|n| !remote || matches!(n.target, Target::Remote(_)))
.map(|n| n.key.clone())
.collect()
};
if same {
self.record(json!({"kind": "restored", "origin": origin, "held": keys(false)}));
} else {
self.record(json!({"kind": "restored", "origin": origin, "paused": keys(true)}));
}
}
/// Wait for any run or drill to end, and start none while the guard
/// lives: books being replaced by a restore push nothing more.
pub async fn pause(&self) -> tokio::sync::MutexGuard<'_, ()> {
self.running.lock().await
}
/// The books' head now.
pub async fn head(&self) -> Option<(u64, String)> {
let store = self.store.clone();
tokio::task::spawn_blocking(move || store.with_store(|s| s.head()).ok())
.await
.ok()
.flatten()
}
/// Back the books up now (INVARIANTS 1, 2): `why` is recorded (a
/// write, open, close, asked). The status after.
pub async fn run(&self, why: &str) -> Result<Status, String> {
self.run_with(why, false).await
}
/// [`Self::run`]; with `new_disk`, a folder target that took these
/// books' backups and holds none now is begun again there (the owner
/// says it is a new disk).
pub async fn run_with(&self, why: &str, new_disk: bool) -> Result<Status, String> {
let _one = self.running.lock().await;
if self.named().await.is_empty() {
return Ok(self.status().await);
}
let (store, dir, book) = (self.store.clone(), self.dir.clone(), self.book.clone());
let made = tokio::task::spawn_blocking(move || {
let git = Git::find()?;
// The stored files first, outside the lock: they never change.
// Only a head start (a file being written can trip it): the
// commit stages them all again.
let _ = backup::stage_blobs(&git, &dir);
// The commit, under it.
store
.with_store(|s| -> Result<_, BackupError> {
let made = backup::commit(&git, &dir, &book, s.head())?;
Ok((made, targets(s, &book), s.head()))
})
.map_err(BackupError::Damaged)
.and_then(|r| r)
})
.await
.unwrap_or(Err(BackupError::Damaged("the backup stopped".into())));
let (made, named, head) = match made {
Ok(m) => m,
Err(e) => {
let e = hide(&e.to_string(), &self.dir);
self.record(json!({"kind": "commit", "why": why, "ok": false, "error": e}));
return Err(e);
}
};
if let Some(m) = &made {
self.record(
json!({"kind": "commit", "why": why, "ok": true, "commit": m.commit,
"head": {"seq": m.head.0, "hash": m.head.1}}),
);
}
// Each target, outside the lock; after a write, one that failed
// lately waits out RETRY_AFTER (the others don't wait for it).
let records = self.records();
let now = chrono::Utc::now();
for n in named {
if paused(&n.key, &records) || (why == "write" && failed_lately(&n.key, &records, now))
{
continue;
}
// Its repository is made only where it never took a backup.
let may_create = new_disk || !held_before(&n.key, &records);
let (dir, book, t) = (self.dir.clone(), self.book.clone(), n.target.clone());
let pushed = tokio::task::spawn_blocking(move || {
let git = Git::find()?;
backup::push(&git, &dir, &book, &t, may_create)
})
.await
.unwrap_or(Err(BackupError::Damaged("the push stopped".into())));
match pushed {
Ok(commit) => self.record(
json!({"kind": "push", "why": why, "target": n.target.kind(),
"at": n.at, "key": n.key, "ok": true, "commit": commit,
"head": {"seq": head.0, "hash": head.1}}),
),
Err(e) => self.record(
json!({"kind": "push", "why": why, "target": n.target.kind(),
"at": n.at, "key": n.key, "ok": false, "error": e.to_string()}),
),
}
}
Ok(self.status().await)
}
/// Whether a drill is due (INVARIANT 5).
pub async fn drill_due(&self) -> bool {
let records = self.records();
let now = chrono::Utc::now();
self.named()
.await
.iter()
.any(|n| drill_due(&n.key, &records, now))
}
/// Drill each target whose drill is due (INVARIANT 5), each recorded.
pub async fn drill(&self) {
let _one = self.running.lock().await;
let records = self.records();
let now = chrono::Utc::now();
for n in self.named().await {
if paused(&n.key, &records) || !drill_due(&n.key, &records, now) {
continue;
}
let (dir, book, t) = (self.dir.clone(), self.book.clone(), n.target.clone());
let drilled = tokio::task::spawn_blocking(move || {
let scratch = ledger_store::ops::ops_dir(&dir).join(format!("drill-{}", t.kind()));
let _ = std::fs::remove_dir_all(&scratch);
let git = Git::find()?;
backup::drill(
&git,
&t,
&book,
&dir.join(ledger_store::JOURNAL_FILE),
&scratch,
)
})
.await
.unwrap_or(Err(BackupError::Damaged("the drill stopped".into())));
match drilled {
Ok(head) => {
self.record(json!({"kind": "drill", "target": n.target.kind(),
"at": n.at, "key": n.key, "ok": true, "head": {"seq": head.0, "hash": head.1}}))
}
Err(e) => self.record(json!({"kind": "drill", "target": n.target.kind(),
"at": n.at, "key": n.key, "ok": false, "error": e.to_string()})),
}
}
}
/// The books' backup standing (INVARIANT 3).
pub async fn status(&self) -> Status {
let store = self.store.clone();
let (book, dir) = (self.book.clone(), self.dir.clone());
tokio::task::spawn_blocking(move || store.with_store(|s| standing(s, &book, &dir)).ok())
.await
.ok()
.flatten()
.unwrap_or_else(|| standing_of(&[], &[], 0, chrono::Utc::now()))
}
/// The standing, read from `store` (held by the caller: books.start).
pub fn standing(&self, store: &ledger_store::Store) -> Status {
standing(store, &self.book, &self.dir)
}
}
fn standing(store: &ledger_store::Store, book: &str, dir: &Path) -> Status {
let records =
ledger_store::chain::read(&ledger_store::ops::ops_dir(dir).join(LOG_FILE), GENESIS)
.map(|r| r.records)
.unwrap_or_default();
standing_of(
&targets(store, book),
&records,
store.head().0,
chrono::Utc::now(),
)
}
fn standing_of(
named: &[Named],
records: &[Map<String, Value>],
head_seq: u64,
now: chrono::DateTime<chrono::Utc>,
) -> Status {
Status {
head_seq,
note: named.is_empty().then(|| {
"no backups are set up: set backup.git_remote (a private repository) and backup.folder (a folder on another disk) with books.set_setting".to_string()
}),
targets: named
.iter()
.map(|n| status_of(n.target.kind(), &n.at, &n.key, records, head_seq, now))
.collect(),
}
}
impl Status {
/// Whether the human should hear of it: none set up, or one stale,
/// failing, or failing its drill.
pub fn needs_attention(&self) -> bool {
self.note.is_some()
|| self
.targets
.iter()
.any(|t| t.stale || t.last_error.is_some() || t.drill_error.is_some())
}
}
/// `text` with the book's folder `dir`, and the data root above it, named
/// in place of their paths (INVARIANT 4); at most ERROR_MAX characters.
fn hide(text: &str, dir: &Path) -> String {
let mut text = text.to_string();
let real = std::fs::canonicalize(dir).unwrap_or_else(|_| dir.to_path_buf());
for d in [dir, real.as_path()] {
text = text.replace(&d.display().to_string(), "the book's folder");
if let Some(root) = d.parent().and_then(Path::parent) {
text = text.replace(&root.display().to_string(), ledger_store::root::THE_ROOT);
}
}
text.chars().take(ERROR_MAX).collect()
}
fn is(r: &Map<String, Value>, kind: &str) -> bool {
r.get("kind").and_then(Value::as_str) == Some(kind)
}
fn ok(r: &Map<String, Value>) -> bool {
r.get("ok") == Some(&Value::Bool(true))
}
fn of(r: &Map<String, Value>, key: &str) -> bool {
r.get("key").and_then(Value::as_str) == Some(key)
}
fn ts(r: &Map<String, Value>) -> Option<chrono::DateTime<chrono::Utc>> {
r.get("ts")
.and_then(Value::as_str)
.and_then(|t| chrono::DateTime::parse_from_rfc3339(t).ok())
.map(|t| t.with_timezone(&chrono::Utc))
}
fn text(r: &Map<String, Value>, field: &str) -> Option<String> {
r.get(field).and_then(Value::as_str).map(str::to_string)
}
/// What a damaged record knew, up to its first line that can't be read at
/// all (a line read but out of the chain still counts: what it adds only
/// holds more back), of whose backups the
/// targets hold: its restored notes, and each target that took a backup
/// (as held).
fn known_before_damage(path: &Path) -> Vec<Value> {
let text = std::fs::read(path).unwrap_or_default();
let mut held: Vec<String> = Vec::new();
let mut out = Vec::new();
for line in text.split(|b| *b == b'\n') {
let Ok(Value::Object(r)) = serde_json::from_slice::<Value>(line) else {
break;
};
if is(&r, "push") && ok(&r) {
if let Some(k) = text_field(&r, "key")
&& !held.contains(&k)
{
held.push(k);
}
} else if is(&r, "restored") {
let mut r = r.clone();
for k in ["seq", "ts", "prev", "hash"] {
r.remove(k);
}
out.push(Value::Object(r));
}
}
if !held.is_empty() {
out.push(json!({"kind": "restored", "carried": true, "held": held}));
}
out
}
fn text_field(r: &Map<String, Value>, field: &str) -> Option<String> {
r.get(field).and_then(Value::as_str).map(str::to_string)
}
/// Whether the target `key` is paused for books restored beside others
/// (INVARIANT 7).
fn paused(key: &str, records: &[Map<String, Value>]) -> bool {
records.iter().any(|r| {
is(r, "restored")
&& r.get("paused")
.and_then(Value::as_array)
.is_some_and(|keys| keys.iter().any(|k| k.as_str() == Some(key)))
})
}
/// Whether the target `key` took a backup of these books (INVARIANT 6).
fn held_before(key: &str, records: &[Map<String, Value>]) -> bool {
records.iter().any(|r| {
(is(r, "push") && of(r, key) && ok(r))
|| (is(r, "restored")
&& r.get("held")
.and_then(Value::as_array)
.is_some_and(|keys| keys.iter().any(|k| k.as_str() == Some(key))))
})
}
/// What books restored beside others are told of the repository paused
/// for them.
pub const PAUSED: &str = "not backed up here: these books were restored as a copy of the books this repository keeps, and backing up here would stop theirs; name a repository of their own (backup.git_remote)";
/// Whether the target `key`'s last push failed less than RETRY_AFTER ago.
fn failed_lately(
key: &str,
records: &[Map<String, Value>],
now: chrono::DateTime<chrono::Utc>,
) -> bool {
records
.iter()
.rfind(|r| is(r, "push") && of(r, key))
.filter(|r| !ok(r))
.and_then(ts)
.is_some_and(|at| {
now.signed_duration_since(at)
< chrono::Duration::from_std(RETRY_AFTER).unwrap_or(chrono::Duration::zero())
})
}
/// Whether the target `key`'s drill is due (INVARIANT 5).
fn drill_due(
key: &str,
records: &[Map<String, Value>],
now: chrono::DateTime<chrono::Utc>,
) -> bool {
if !records.iter().any(|r| is(r, "push") && of(r, key) && ok(r)) {
return false;
}
match records.iter().rfind(|r| is(r, "drill") && of(r, key)) {
None => true,
Some(d) => ts(d).is_none_or(|at| {
let wait = if ok(d) {
DRILL_EVERY
} else {
chrono::Duration::from_std(RETRY_AFTER).unwrap_or(DRILL_EVERY)
};
now.signed_duration_since(at) >= wait
}),
}
}
/// A target's standing from the backup record (INVARIANT 3).
fn status_of(
kind: &str,
at: &str,
key: &str,
records: &[Map<String, Value>],
head_seq: u64,
now: chrono::DateTime<chrono::Utc>,
) -> TargetStatus {
let last_ok = records
.iter()
.rposition(|r| is(r, "push") && of(r, key) && ok(r));
let through_seq = last_ok
.and_then(|i| records[i].get("head"))
.and_then(|h| h.get("seq"))
.and_then(Value::as_u64);
// Since its last good push (or, never having had one, since its first
// try; or, never tried, the failed commits that kept it from one):
// this target's tries, and the commits.
let from = match last_ok {
Some(i) => i + 1,
None => records.iter().position(|r| of(r, key)).unwrap_or_else(|| {
records
.iter()
.rposition(|r| !(is(r, "commit") && !ok(r)))
.map_or(0, |i| i + 1)
}),
};
let since: Vec<&Map<String, Value>> = records[from..]
.iter()
.filter(|r| is(r, "commit") || (is(r, "push") && of(r, key)))
.collect();
let behind = through_seq.is_none_or(|s| s < head_seq);
let old = since
.first()
.and_then(|r| ts(r))
.is_some_and(|t| now.signed_duration_since(t) > STALE_AFTER);
let last_error = since
.iter()
.rev()
.find(|r| !ok(r) || is(r, "push"))
.filter(|r| !ok(r))
.and_then(|r| text(r, "error"));
let drill = records.iter().rfind(|r| is(r, "drill") && of(r, key));
let paused = paused(key, records);
TargetStatus {
target: kind.to_string(),
at: at.to_string(),
last_ok_at: last_ok.and_then(|i| text(&records[i], "ts")),
through_seq,
stale: behind && old && head_seq > 0 && !paused,
last_error: if paused {
Some(PAUSED.to_string())
} else {
last_error
},
last_drill_at: drill.and_then(|r| text(r, "ts")),
drill_error: drill.filter(|r| !ok(r)).and_then(|r| text(r, "error")),
paused,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn no_error_names_the_root() {
let dir = Path::new("/home/u/.local/share/wanderbooks/books/acme");
assert_eq!(
hide(
"fatal: '/home/u/.local/share/wanderbooks/books/acme/archive.git' and /home/u/.local/share/wanderbooks/run",
dir
),
format!(
"fatal: 'the book's folder/archive.git' and {}/run",
ledger_store::root::THE_ROOT
)
);
assert_eq!(hide(&"x".repeat(2000), dir).len(), ERROR_MAX);
}
/// A record of `kind` for target `key`, `ago` before `now`.
fn rec(
now: chrono::DateTime<chrono::Utc>,
kind: &str,
key: &str,
ok: bool,
seq: u64,
ago: chrono::Duration,
) -> Map<String, Value> {
let mut m = Map::new();
m.insert("kind".into(), json!(kind));
if kind != "commit" {
m.insert("key".into(), json!(key));
}
m.insert("ok".into(), json!(ok));
m.insert("ts".into(), json!((now - ago).to_rfc3339()));
if ok {
m.insert("head".into(), json!({"seq": seq, "hash": "h"}));
} else {
m.insert("error".into(), json!(format!("{kind} failed")));
}
m
}
#[test]
fn a_target_is_stale_only_when_behind_since_a_day() {
let now = chrono::Utc::now();
let r = |kind, ok, seq, ago| rec(now, kind, "k", ok, seq, ago);
let st = |records: &[Map<String, Value>], head| {
status_of("folder", "f", "k", records, head, now)
};
let day = chrono::Duration::hours(24);
let hour = chrono::Duration::hours(1);
// Nothing tried yet: not stale.
assert!(!st(&[], 0).stale);
assert!(!st(&[], 4).stale);
// Never pushed, the first try a day ago: stale; an hour ago: not.
assert!(
st(
&[
r("commit", true, 3, day + hour),
r("push", false, 0, day + hour)
],
3
)
.stale
);
assert!(!st(&[r("commit", true, 3, hour), r("push", false, 0, hour)], 3).stale);
// Pushed long ago, and quiet since: a new write isn't stale at once.
let quiet = [r("commit", true, 3, day * 3), r("push", true, 3, day * 3)];
assert!(!st(&quiet, 4).stale);
// ... nor while it holds the head.
assert!(!st(&quiet, 3).stale);
// Behind since a commit a day old, its pushes failing: stale.
let failing = [
r("push", true, 3, day * 3),
r("commit", true, 4, day + hour),
r("push", false, 0, day),
r("push", false, 0, hour),
];
let t = st(&failing, 4);
assert!(t.stale);
assert_eq!(
(t.through_seq, t.last_error.as_deref()),
(Some(3), Some("push failed"))
);
// A good push ends the error.
let mended = [
&failing[..],
&[r("push", true, 4, chrono::Duration::zero())],
]
.concat();
let t = st(&mended, 4);
assert_eq!(
(t.stale, t.last_error, t.through_seq),
(false, None, Some(4))
);
// A failed commit is every target's error, until a good push.
let t = st(&[r("push", true, 3, day), r("commit", false, 0, hour)], 4);
assert_eq!(t.last_error.as_deref(), Some("commit failed"));
// A good commit after a failed push doesn't hide the failure.
let t = st(&[r("push", false, 0, hour), r("commit", true, 4, hour)], 4);
assert_eq!(t.last_error.as_deref(), Some("push failed"));
// Another target's records are not its: a new folder starts clean.
let other = [
rec(now, "push", "other", true, 3, hour),
rec(now, "push", "other", false, 0, hour),
];
let t = st(&other, 3);
assert_eq!(
(t.through_seq, t.last_error, t.last_ok_at),
(None, None, None)
);
}
#[test]
fn a_target_failing_lately_is_left_out_of_a_write_alone() {
let now = chrono::Utc::now();
let retry = chrono::Duration::from_std(RETRY_AFTER).unwrap();
let second = chrono::Duration::seconds(1);
let failed = |ago| rec(now, "push", "k", false, 0, ago);
assert!(failed_lately("k", &[failed(retry - second)], now));
assert!(!failed_lately("k", &[failed(retry)], now));
assert!(!failed_lately(
"k",
&[failed(second), rec(now, "push", "k", true, 3, second)],
now
));
assert!(!failed_lately("other", &[failed(second)], now));
assert!(!failed_lately("k", &[], now));
}
#[test]
fn commits_failing_from_the_start_are_every_targets_trouble() {
let now = chrono::Utc::now();
let day = chrono::Duration::hours(24);
let hour = chrono::Duration::hours(1);
// git missing (say) from the first run on: never a push of its own.
let failing = [
rec(now, "commit", "", false, 0, day + hour),
rec(now, "commit", "", false, 0, hour),
];
let t = status_of("folder", "f", "k", &failing, 3, now);
assert_eq!(t.last_error.as_deref(), Some("commit failed"));
assert!(t.stale, "{t:?}");
// Failing an hour only: said, not yet stale.
let t = status_of("folder", "f", "k", &failing[1..], 3, now);
assert_eq!(
(t.last_error.as_deref(), t.stale),
(Some("commit failed"), false)
);
// A commit that worked since (its push still to come): no error.
let mended = [
&failing[..],
&[rec(now, "commit", "", true, 3, chrono::Duration::zero())],
]
.concat();
assert_eq!(
status_of("folder", "f", "k", &mended, 3, now).last_error,
None
);
}
#[test]
fn restored_books_know_which_backups_were_theirs() {
let now = chrono::Utc::now();
let restored = |field: &str, keys: &[&str]| {
let mut m = Map::new();
m.insert("kind".into(), json!("restored"));
m.insert(field.into(), json!(keys));
m.insert("ts".into(), json!(now.to_rfc3339()));
m
};
// The same books: their targets took their backups.
let same = [restored("held", &["k"])];
assert!(held_before("k", &same));
assert!(!held_before("new", &same));
assert!(!paused("k", &same));
// Others' (restored beside them): their repository is paused, said.
let beside = [restored("paused", &["r"])];
assert!(paused("r", &beside));
assert!(!paused("k", &beside));
assert!(!held_before("r", &beside));
let t = status_of("repository", "x", "r", &beside, 9, now);
assert_eq!(t.last_error.as_deref(), Some(PAUSED));
assert!(!t.stale && t.paused);
// A push of its own counts as held, too.
assert!(held_before(
"k",
&[rec(now, "push", "k", true, 3, chrono::Duration::zero())]
));
assert!(!held_before(
"k",
&[rec(now, "push", "k", false, 0, chrono::Duration::zero())]
));
}
#[test]
fn a_target_named_later_starts_its_own_clock() {
let now = chrono::Utc::now();
let day = chrono::Duration::hours(24);
let hour = chrono::Duration::hours(1);
// Commits for a week, to another target; then this one's first try.
let records = [
rec(now, "commit", "", true, 2, day * 7),
rec(now, "push", "old", true, 2, day * 7),
rec(now, "commit", "", true, 3, day * 2),
rec(now, "push", "k", false, 0, hour),
];
let t = status_of("folder", "f", "k", &records, 3, now);
assert!(!t.stale, "{t:?}");
assert_eq!(t.last_error.as_deref(), Some("push failed"));
// Holding the head: never stale; one behind since long: stale.
let held = [
rec(now, "push", "k", true, 3, day * 3),
rec(now, "push", "k", false, 0, day * 2),
];
assert!(!status_of("folder", "f", "k", &held, 3, now).stale);
assert!(status_of("folder", "f", "k", &held, 4, now).stale);
}
#[test]
fn drills_are_due_weekly_after_a_pass_and_soon_after_a_failure() {
let now = chrono::Utc::now();
let r = |kind, ok, ago| rec(now, kind, "k", ok, 3, ago);
let hour = chrono::Duration::hours(1);
let week = DRILL_EVERY;
let retry = chrono::Duration::from_std(RETRY_AFTER).unwrap();
// Nothing backed up there: nothing to drill.
assert!(!drill_due("k", &[], now));
assert!(!drill_due("k", &[r("push", false, hour)], now));
// Backed up, never drilled: due.
let pushed = r("push", true, hour);
assert!(drill_due("k", std::slice::from_ref(&pushed), now));
// Passed: due a week after.
assert!(!drill_due(
"k",
&[pushed.clone(), r("drill", true, week - hour)],
now
));
assert!(drill_due(
"k",
&[pushed.clone(), r("drill", true, week)],
now
));
// Failed: tried again RETRY_AFTER after.
let t = drill_due(
"k",
&[
pushed.clone(),
r("drill", false, retry - chrono::Duration::seconds(1)),
],
now,
);
assert!(!t);
assert!(drill_due(
"k",
&[pushed.clone(), r("drill", false, retry)],
now
));
// Another target's drill isn't its, nor its push.
assert!(drill_due(
"k",
&[pushed.clone(), rec(now, "drill", "o", true, 3, hour)],
now
));
assert!(!drill_due(
"k",
&[rec(now, "push", "o", true, 3, hour)],
now
));
// A failed drill is said until one passes.
let t = status_of(
"folder",
"f",
"k",
&[pushed.clone(), r("drill", false, hour)],
3,
now,
);
assert_eq!(t.drill_error.as_deref(), Some("drill failed"));
assert!(t.last_drill_at.is_some());
let t = status_of(
"folder",
"f",
"k",
&[pushed, r("drill", false, hour), r("drill", true, hour)],
3,
now,
);
assert_eq!(t.drill_error, None);
}
#[test]
fn each_kind_of_trouble_needs_attention_alone() {
let clean = TargetStatus {
target: "folder".into(),
at: "f".into(),
last_ok_at: Some("t".into()),
through_seq: Some(3),
stale: false,
last_error: None,
last_drill_at: None,
drill_error: None,
paused: false,
};
let status = |t: TargetStatus| Status {
head_seq: 3,
targets: vec![t],
note: None,
};
assert!(!status(clean.clone()).needs_attention());
assert!(
status(TargetStatus {
stale: true,
..clean.clone()
})
.needs_attention()
);
assert!(
status(TargetStatus {
last_error: Some("e".into()),
..clean.clone()
})
.needs_attention()
);
assert!(
status(TargetStatus {
drill_error: Some("e".into()),
..clean.clone()
})
.needs_attention()
);
let none = Status {
head_seq: 3,
targets: vec![],
note: Some("n".into()),
};
assert!(none.needs_attention());
}
}
+1
View File
@@ -81,6 +81,7 @@ pub const SECTIONS: &[(&str, &str)] = &[
include_str!("../handbook/settings-and-conventions.md"),
),
("viewer", include_str!("../handbook/viewer.md")),
("backups", include_str!("../handbook/backups.md")),
(
"rules-that-change",
include_str!("../handbook/rules-that-change.md"),
+1
View File
@@ -16,6 +16,7 @@
pub mod access;
pub mod admin;
pub mod backup;
mod handbook;
#[cfg(unix)]
pub mod local;
+475 -2
View File
@@ -75,7 +75,7 @@ use std::os::fd::AsRawFd;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex, RwLock};
use std::time::Duration;
use std::time::{Duration, Instant};
use tokio_util::sync::CancellationToken;
use tokio_util::task::TaskTracker;
@@ -103,6 +103,18 @@ pub const REFUSED_HEADER: &str = "x-wanderbooks-refused";
/// How often each book's lapsed uploads are swept.
const SWEEP_EVERY: Duration = Duration::from_secs(10);
/// How long the books stay unwritten before they are backed up.
const BACKUP_QUIET: Duration = Duration::from_secs(30);
/// How often the books are looked at for a backup due.
const BACKUP_LOOK: Duration = Duration::from_secs(2);
/// How often each book is looked at for a drill due.
const DRILL_LOOK: Duration = Duration::from_secs(3600);
/// How long after books are first seen open their first drill is looked
/// for (a drill that fell due while they were closed): long enough that a
/// backup on opening comes first.
const DRILL_FIRST: Duration = Duration::from_secs(60);
/// The most the last backup at a close may take.
const BACKUP_AT_CLOSE: Duration = Duration::from_secs(60);
type Service = StreamableHttpService<BooksServer, LocalSessionManager>;
@@ -122,6 +134,8 @@ enum Book {
uploads: ledger_store::uploads::Uploads,
/// The files it made for agents (`/b/{id}/x/{export}`).
exports: ledger_store::exports::Exports,
/// Its backups (backup.rs).
backups: Arc<crate::backup::Backups>,
},
/// Not opened, with why (INVARIANT 3).
Failed(String),
@@ -160,6 +174,14 @@ struct Books {
/// Which token opened each MCP session: a session is used only with
/// the token that opened it (INVARIANT 8).
session_tokens: Mutex<BTreeMap<String, String>>,
/// How long the books stay unwritten before a backup is made
/// (`--backup-quiet`, for tests; else BACKUP_QUIET).
backup_quiet: Duration,
/// How long after books are first seen open a drill is first looked
/// for (`--drill-first`, for tests; else DRILL_FIRST).
drill_first: Duration,
/// Held through a restore: one at a time.
restoring: Mutex<()>,
}
impl Books {
@@ -284,7 +306,17 @@ impl Books {
// INVARIANT 2): a viewer can show its presence from the start.
ledger_store::ops::prepare(store.data_dir()).map_err(|e| self.hide(e.to_string()))?;
let live = Arc::new(Mutex::new(live));
let book_dir = store.data_dir().to_path_buf();
let book_id = book_dir
.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_default();
let store = SharedStore::new(store);
let backups = Arc::new(
crate::backup::Backups::open(&book_id, &book_dir, store.clone())
.map_err(|e| self.hide(e))?,
);
let server_backups = backups.clone();
let log = access.clone();
let shared = store.clone();
let config = StreamableHttpServerConfig::default()
@@ -301,7 +333,8 @@ impl Books {
.with_layout(layout.clone())
.with_access(log.clone())
.with_uploads(server_uploads.clone())
.with_exports(server_exports.clone()),
.with_exports(server_exports.clone())
.with_backups(server_backups.clone()),
)
},
Arc::new(sessions()),
@@ -313,6 +346,7 @@ impl Books {
access,
uploads,
exports,
backups,
})
}
@@ -449,6 +483,13 @@ impl Books {
Err(e) => refused(e),
}
}
Request::Restore {
id,
source,
replace,
confirmed,
expect_head,
} => self.restore(id, source, replace, confirmed, expect_head),
Request::Adopt {
id,
source,
@@ -589,6 +630,119 @@ impl Books {
}
}
impl Books {
/// Restore a backup (admin INVARIANT 4; ledger_store::root INVARIANT
/// 7): a preview, or the books restored and opened. Books it replaces
/// are let go first, and opened again if the restore fails.
fn restore(
&self,
id: String,
source: String,
replace: bool,
confirmed: bool,
expect_head: Option<(u64, String)>,
) -> Response {
// admin INVARIANT 3: never the root, nor the address given.
let said = |text: String| {
let text = if source.chars().count() > 1 {
text.replace(&source, "the backup given")
} else {
text
};
Response::Refused(self.hide(text))
};
let id = match BookId::parse(&id) {
Ok(id) => id,
Err(e) => return said(e.to_string()),
};
let _one = self.restoring.lock().unwrap_or_else(|e| e.into_inner());
let git = match ledger_store::backup::Git::find() {
Ok(git) => git,
Err(e) => return said(e.to_string()),
};
// A preview only reports (whether books of that id exist among
// what it says): the restore itself is refused over books, or
// replacing none.
if !confirmed {
return match self.root.restore_preview(&git, &source, &id) {
Ok(p) => Response::RestorePreview(admin::RestorePreview {
head: p.head,
blobs: p.blobs,
vault_files: p.vault_files,
exists: p.exists,
origin: p.origin,
}),
Err(e) => said(e.to_string()),
};
}
let Some(expect_head) = expect_head else {
return said("a restore is confirmed against the preview's head: preview first".into());
};
let held = self.held().contains_key(id.as_str());
if replace && !held {
return said(format!(
"there are no books called {:?} to replace: nothing was restored",
id.as_str()
));
}
if !replace && held {
return said(format!(
"there are already books called {:?}: restore them under another id (--id), or replace them (--replace; they are kept in legacy/)",
id.as_str()
));
}
// The books replaced are let go (their writer lock released),
// their backups idle until the restore is done: none of theirs
// reaches a target after the backup is read.
let old = if replace {
self.held
.write()
.unwrap_or_else(|e| e.into_inner())
.remove(id.as_str())
} else {
None
};
let old_backups = match &old {
Some(Book::Open { backups, .. }) => Some(backups.clone()),
_ => None,
};
let rt = tokio::runtime::Handle::current();
// Its requests end first (a books.back_up_now among them waits for
// a run of the loop's), then its last run.
if let Some(Book::Open { store, .. }) = &old {
rt.block_on(store.let_go());
}
let _paused = old_backups.as_ref().map(|b| rt.block_on(b.pause()));
match self.root.restore(&git, &source, &id, &expect_head, replace) {
Ok(r) => {
// Restored: kept whether or not it opens; its backups told
// whose books these were (backup.rs INVARIANTS 6, 7).
let book = self.open(&id);
let problem = book.problem();
self.hold(&id, book);
Response::Restored(admin::Restored {
id: r.id.to_string(),
head: r.head,
blobs_verified: r.blobs_verified,
vault_files: r.vault_files,
replaced: r.replaced,
origin: r.origin,
problem,
})
}
Err(e) => {
// The books it would have replaced, let go above, open
// again (only this restore runs, so nothing else holds them).
if replace && !self.held().contains_key(id.as_str()) {
let book = self.open(&id);
self.hold(&id, book);
}
said(e.to_string())
}
}
}
}
/// The settings a settings file moves in at adoption, checked as
/// `books.import_settings` checks them: they parse, they apply, and the
/// vault key file it names (which the books don't keep) is the one given.
@@ -1603,10 +1757,28 @@ pub fn run(
args: impl IntoIterator<Item = String>,
) -> Result<(), Box<dyn std::error::Error>> {
let mut root: Option<PathBuf> = None;
let mut backup_quiet = BACKUP_QUIET;
let mut drill_first = DRILL_FIRST;
let mut args = args.into_iter();
while let Some(arg) = args.next() {
match arg.as_str() {
"--root" => root = Some(PathBuf::from(args.next().ok_or("--root needs a folder")?)),
// How long the books stay unwritten before a backup (tests).
"--backup-quiet" => {
let ms: u64 = args
.next()
.and_then(|v| v.parse().ok())
.ok_or("--backup-quiet needs milliseconds")?;
backup_quiet = Duration::from_millis(ms);
}
// How soon after books open a drill is first looked for (tests).
"--drill-first" => {
let ms: u64 = args
.next()
.and_then(|v| v.parse().ok())
.ok_or("--drill-first needs milliseconds")?;
drill_first = Duration::from_millis(ms);
}
"--help" | "-h" => {
print_usage(prog);
return Ok(());
@@ -1684,6 +1856,9 @@ pub fn run(
started: std::time::Instant::now(),
principals: RwLock::new(Err(String::new())),
session_tokens: Mutex::new(BTreeMap::new()),
backup_quiet,
drill_first,
restoring: Mutex::new(()),
});
{
let loaded = books.load_principals();
@@ -1753,6 +1928,10 @@ async fn serve(
}
let held: Vec<String> = books.held().keys().cloned().collect();
eprintln!("wanderbooks server: ready, holding {} books", held.len());
// Every open book is backed up when it opens, once its writing has
// stopped for a while, and again after a failed run (backup.rs).
let (backing, backup_stop) = (books.clone(), stop.clone());
let backup_loop = tokio::spawn(async move { keep_backed_up(backing, backup_stop).await });
// Every open book's lapsed uploads go within SWEEP_EVERY of lapsing,
// whatever else the server is doing (uploads.rs INVARIANT 3).
let (sweeping, sweep_stop) = (books.clone(), stop.clone());
@@ -1823,6 +2002,22 @@ async fn serve(
for store in books.stores() {
store.close().await;
}
// A last backup of what was written since the last one, for a while
// at most: the books are closing, nothing more is written. The loop
// stops at its next step (`stop`); a run of its still under way
// finishes first (each book runs one at a time).
drop(backup_loop);
let held: Vec<Arc<crate::backup::Backups>> = books
.held()
.values()
.filter_map(|b| match b {
Book::Open { backups, .. } => Some(backups.clone()),
Book::Failed(_) => None,
})
.collect();
for b in held {
let _ = tokio::time::timeout(BACKUP_AT_CLOSE, b.run("close")).await;
}
// Answers on their way out are sent, for a while: a client sending
// a request it never finishes doesn't hold the books open.
let answers = async {
@@ -1916,10 +2111,288 @@ fn watch_for_close(stops: &Stops) -> std::io::Result<()> {
Ok(())
}
/// What the backup loop knows of one open book.
struct Seen {
/// Its backups, by identity: books restored over others are new.
backups: Arc<crate::backup::Backups>,
watch: Watch,
}
/// One open book's backups, as the loop has watched them.
#[derive(Debug, Clone, Copy)]
struct Watch {
/// Its head's seq when last looked at, and when that changed.
seq: u64,
moved_at: Instant,
/// The seq last committed and offered to the targets.
backed: Option<u64>,
/// When the last run's commit failed, if it did: nothing runs again
/// for RETRY_AFTER.
failed_at: Option<Instant>,
/// When the last run left a target failing, if it did: it is tried
/// again RETRY_AFTER on, written to or not.
target_failed_at: Option<Instant>,
/// When a drill is next looked for.
drill_next: Instant,
}
impl Watch {
/// Books first seen open now, at `seq`: a drill is first looked for
/// `drill_first` from now.
fn new(seq: u64, now: Instant, drill_first: Duration) -> Self {
Watch {
seq,
moved_at: now,
backed: None,
failed_at: None,
target_failed_at: None,
drill_next: now + drill_first,
}
}
/// A run through `seq` came to `outcome` at `now`: None when its
/// commit failed, else whether every target holds it.
fn ran(&mut self, seq: u64, outcome: Option<bool>, now: Instant) {
match outcome {
None => self.failed_at = Some(now),
Some(all) => {
self.backed = Some(seq);
self.failed_at = None;
self.target_failed_at = (!all).then_some(now);
}
}
}
}
/// Why a backup is due now, if one is: the books just seen open; or
/// written and quiet for `quiet` since (not yet backed up); or a target
/// left failing RETRY_AFTER ago. Nothing within RETRY_AFTER of a failed
/// commit.
fn backup_due(
seen: Option<&Watch>,
seq: u64,
now: Instant,
quiet: Duration,
) -> Option<&'static str> {
let Some(s) = seen else {
return Some("open");
};
let after =
|t: Option<Instant>| t.is_some_and(|f| now.duration_since(f) >= crate::backup::RETRY_AFTER);
if s.failed_at.is_some() && !after(s.failed_at) {
return None;
}
let quiet_now = if s.seq == seq {
now.duration_since(s.moved_at) >= quiet
} else {
quiet.is_zero()
};
if s.backed != Some(seq) && quiet_now {
return Some("write");
}
after(s.target_failed_at).then_some("retry")
}
/// Whether a run's every target holds the books: a target paused for
/// these books (restored as a copy of others) is no failure to try again.
fn every_target_held(status: &crate::backup::Status) -> bool {
status
.targets
.iter()
.all(|t| t.paused || t.last_error.is_none())
}
/// Whether to look for a drill due now (then again DRILL_LOOK on).
fn drill_look_due(seen: &Watch, now: Instant) -> bool {
now >= seen.drill_next
}
/// Back every open book up (backup.rs) whenever [`backup_due`] says, and
/// drill its backups when a drill is due (looked for every DRILL_LOOK).
/// Until `stop`.
async fn keep_backed_up(books: Arc<Books>, stop: CancellationToken) {
let mut seen: BTreeMap<String, Seen> = BTreeMap::new();
let mut tick = tokio::time::interval(BACKUP_LOOK);
loop {
tokio::select! {
() = stop.cancelled() => return,
_ = tick.tick() => {}
}
let open: Vec<(String, Arc<crate::backup::Backups>)> = books
.held()
.iter()
.filter_map(|(id, b)| match b {
Book::Open { backups, .. } => Some((id.clone(), backups.clone())),
Book::Failed(_) => None,
})
.collect();
seen.retain(|id, _| open.iter().any(|(o, _)| o == id));
for (id, backups) in open {
if stop.is_cancelled() {
return;
}
let Some((seq, _)) = backups.head().await else {
continue;
};
let now = Instant::now();
// Books restored over others are seen anew.
if seen
.get(&id)
.is_some_and(|s| !Arc::ptr_eq(&s.backups, &backups))
{
seen.remove(&id);
}
let due = backup_due(
seen.get(&id).map(|s| &s.watch),
seq,
now,
books.backup_quiet,
);
let s = &mut seen
.entry(id)
.or_insert(Seen {
backups: backups.clone(),
watch: Watch::new(seq, now, books.drill_first),
})
.watch;
if s.seq != seq {
s.seq = seq;
s.moved_at = now;
}
if let Some(why) = due {
let outcome = backups
.run(why)
.await
.ok()
.map(|status| every_target_held(&status));
s.ran(seq, outcome, now);
}
if stop.is_cancelled() {
return;
}
// Drills (backup.rs INVARIANT 5).
if drill_look_due(s, now) {
s.drill_next = now + DRILL_LOOK;
if backups.drill_due().await {
backups.drill().await;
}
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_backup_is_due_on_open_then_once_quiet_after_a_write() {
let t0 = Instant::now();
let quiet = Duration::from_secs(30);
let s = |seq, moved: u64, backed, failed: Option<u64>| Watch {
seq,
moved_at: t0 + Duration::from_secs(moved),
backed,
failed_at: failed.map(|f| t0 + Duration::from_secs(f)),
target_failed_at: None,
drill_next: t0,
};
let at = |secs: u64| t0 + Duration::from_secs(secs);
// First seen: at once.
assert_eq!(backup_due(None, 5, t0, quiet), Some("open"));
// Backed up through the head: nothing.
assert_eq!(
backup_due(Some(&s(5, 0, Some(5), None)), 5, at(100), quiet),
None
);
// Written: not until it has been quiet `quiet`.
assert_eq!(
backup_due(Some(&s(5, 0, Some(5), None)), 6, at(1), quiet),
None
);
assert_eq!(
backup_due(Some(&s(6, 10, Some(5), None)), 6, at(39), quiet),
None
);
assert_eq!(
backup_due(Some(&s(6, 10, Some(5), None)), 6, at(40), quiet),
Some("write")
);
// No quiet asked for: at once.
assert_eq!(
backup_due(Some(&s(5, 0, Some(5), None)), 6, at(1), Duration::ZERO),
Some("write")
);
// A failure waits RETRY_AFTER.
let retry = crate::backup::RETRY_AFTER.as_secs();
let failed = s(6, 10, Some(5), Some(50));
assert_eq!(
backup_due(Some(&failed), 6, at(50 + retry - 1), quiet),
None
);
assert_eq!(
backup_due(Some(&failed), 6, at(50 + retry), quiet),
Some("write")
);
// A drill is first looked for `drill_first` after the books are
// seen open, never at once.
let first = Duration::from_secs(60);
let w = Watch::new(1, t0, first);
assert!(!drill_look_due(&w, t0));
assert!(!drill_look_due(&w, t0 + first - Duration::from_millis(1)));
assert!(drill_look_due(&w, t0 + first));
assert_eq!((w.backed, w.failed_at, w.seq), (None, None, 1));
// A run's outcome: its commit failed (nothing again for a
// while), or made and some target failing (tried again later,
// while writes still go to the others), or all well.
let mut w = Watch::new(1, t0, first);
w.ran(4, None, at(5));
assert_eq!(
(w.backed, w.failed_at, w.target_failed_at),
(None, Some(at(5)), None)
);
w.ran(4, Some(false), at(9));
assert_eq!(
(w.backed, w.failed_at, w.target_failed_at),
(Some(4), None, Some(at(9)))
);
w.seq = 5;
w.moved_at = at(10);
assert_eq!(backup_due(Some(&w), 5, at(10 + 30), quiet), Some("write"));
w.backed = Some(5);
assert_eq!(backup_due(Some(&w), 5, at(9 + retry - 1), quiet), None);
assert_eq!(backup_due(Some(&w), 5, at(9 + retry), quiet), Some("retry"));
// A paused target is no failure; another's error is.
let target = |paused: bool, error: Option<&str>| crate::backup::TargetStatus {
target: "repository".into(),
at: "x".into(),
last_ok_at: None,
through_seq: None,
stale: false,
last_error: error.map(str::to_string),
last_drill_at: None,
drill_error: None,
paused,
};
let status = |targets| crate::backup::Status {
head_seq: 5,
targets,
note: None,
};
assert!(every_target_held(&status(vec![
target(true, Some("paused")),
target(false, None)
])));
assert!(!every_target_held(&status(vec![
target(false, Some("no")),
target(false, None)
])));
assert!(every_target_held(&status(vec![])));
w.ran(5, Some(true), at(9 + retry));
assert_eq!((w.failed_at, w.target_failed_at), (None, None));
assert_eq!(backup_due(Some(&w), 5, at(9 + retry * 3), quiet), None);
}
#[test]
fn a_socket_pair_is_this_account_at_both_ends() {
let (a, b) = std::os::unix::net::UnixStream::pair().unwrap();
+112 -11
View File
@@ -23,6 +23,11 @@
//! with include_tins, export_cpa_package, get_w9) need books:admin,
//! and sealing one into the vault (set_tin, set_w9, which can replace
//! the one there) needs books:post (review of slice 4).
//! 6. Where the books' backups go is books:admin's, as backing up is
//! (a backup is the whole book, its vault and key included):
//! setting a `backup.*` key, or importing settings that hold one,
//! needs books:admin; other settings books:post as before (review of
//! slice 7).
//! ```
use ledger_store::auth::Scope;
@@ -38,9 +43,17 @@ pub enum Needs {
/// books:post for an approval rule (the control that gates posting),
/// books:write for any other convention (INVARIANT 4).
PostForApprovalRules,
/// books:admin for a change to a `backup.*` setting, books:post for
/// any other (INVARIANT 6).
AdminForBackupSettings,
/// As PostWhenConfirmed, and books:admin when the settings imported
/// hold a `[backup]` table (INVARIANT 6).
ImportSettings,
}
use Needs::{Always, PostForApprovalRules, PostWhenConfirmed};
use Needs::{
AdminForBackupSettings, Always, ImportSettings, PostForApprovalRules, PostWhenConfirmed,
};
const R: Needs = Always(Scope::Read);
const W: Needs = Always(Scope::Write);
const P: Needs = Always(Scope::Post);
@@ -49,12 +62,12 @@ const C: Needs = PostWhenConfirmed;
const K: Needs = PostForApprovalRules;
/// Every tool and what it needs (INVARIANT 1).
pub const TABLE: [(&str, Needs); 109] = [
pub const TABLE: [(&str, Needs); 111] = [
("books.handbook", R),
("books.start", R),
// Settings are committed at once: the settings commit of the plan.
("books.set_setting", P),
("books.import_settings", C),
("books.set_setting", AdminForBackupSettings),
("books.import_settings", ImportSettings),
("books.export_settings", R),
("books.conventions", R),
("books.set_convention", K),
@@ -166,6 +179,8 @@ pub const TABLE: [(&str, Needs); 109] = [
("books.profit_and_loss_by_class", R),
("books.export_cpa_package", A),
("books.access_log", A),
("books.backup", R),
("books.back_up_now", A),
];
/// What `tool` needs, given its arguments; None for a tool this server
@@ -178,11 +193,38 @@ pub fn needs(
Some(match needs {
Always(scope) => *scope,
PostWhenConfirmed => {
let confirmed = args
.and_then(|a| a.get("confirmed"))
.and_then(serde_json::Value::as_bool)
.unwrap_or(false);
if confirmed { Scope::Post } else { Scope::Write }
if confirmed(args) {
Scope::Post
} else {
Scope::Write
}
}
AdminForBackupSettings => {
let backup = args
.and_then(|a| a.get("changes"))
.and_then(serde_json::Value::as_array)
.is_some_and(|changes| {
changes.iter().any(|c| {
c.get("key")
.and_then(serde_json::Value::as_str)
.is_some_and(|k| k.trim().to_ascii_lowercase().starts_with("backup."))
})
});
if backup { Scope::Admin } else { Scope::Post }
}
ImportSettings => {
let backup = args
.and_then(|a| a.get("toml_text"))
.and_then(serde_json::Value::as_str)
.and_then(|t| t.parse::<toml::Table>().ok())
.is_some_and(|t| t.contains_key("backup"));
if backup {
Scope::Admin
} else if confirmed(args) {
Scope::Post
} else {
Scope::Write
}
}
PostForApprovalRules => {
let kind = args
@@ -197,15 +239,24 @@ pub fn needs(
})
}
fn confirmed(args: Option<&serde_json::Map<String, serde_json::Value>>) -> bool {
args.and_then(|a| a.get("confirmed"))
.and_then(serde_json::Value::as_bool)
.unwrap_or(false)
}
/// Whether a token with `scopes` may list `tool` (it can call it in some
/// form: a tool that posts when confirmed shows to a books:write token,
/// which may preview).
pub fn shown(tool: &str, scopes: &std::collections::BTreeSet<Scope>) -> bool {
match TABLE.iter().find(|(name, _)| *name == tool) {
Some((_, Always(scope))) => scopes.contains(scope),
Some((_, PostWhenConfirmed | PostForApprovalRules)) => {
Some((_, PostWhenConfirmed | PostForApprovalRules | ImportSettings)) => {
scopes.contains(&Scope::Write) || scopes.contains(&Scope::Post)
}
Some((_, AdminForBackupSettings)) => {
scopes.contains(&Scope::Post) || scopes.contains(&Scope::Admin)
}
None => false,
}
}
@@ -253,6 +304,56 @@ mod tests {
}
}
#[test]
fn where_backups_go_is_admins() {
let args = |v: serde_json::Value| v.as_object().cloned();
let set = |keys: &[&str]| {
let changes: Vec<_> = keys
.iter()
.map(|k| json!({"key": k, "value": "x"}))
.collect();
needs(
"books.set_setting",
args(json!({"changes": changes})).as_ref(),
)
};
assert_eq!(set(&["company.name"]), Some(Scope::Post));
assert_eq!(set(&["backup.folder"]), Some(Scope::Admin));
assert_eq!(
set(&["company.name", "backup.git_remote"]),
Some(Scope::Admin)
);
assert_eq!(set(&[" Backup.Folder"]), Some(Scope::Admin));
assert_eq!(needs("books.set_setting", None), Some(Scope::Post));
let import = |text: &str, confirmed: bool| {
needs(
"books.import_settings",
args(json!({"toml_text": text, "confirmed": confirmed})).as_ref(),
)
};
assert_eq!(
import("[company]\nname = \"A\"\n", false),
Some(Scope::Write)
);
assert_eq!(import("[company]\nname = \"A\"\n", true), Some(Scope::Post));
assert_eq!(
import("[backup]\nfolder = \"/x\"\n", false),
Some(Scope::Admin)
);
assert_eq!(
import("[backup]\nfolder = \"/x\"\n", true),
Some(Scope::Admin)
);
// Shown to whoever can set some setting.
let post: std::collections::BTreeSet<Scope> = [Scope::Post].into();
let write: std::collections::BTreeSet<Scope> = [Scope::Write].into();
let admin: std::collections::BTreeSet<Scope> = [Scope::Admin].into();
assert!(shown("books.set_setting", &post));
assert!(shown("books.set_setting", &admin));
assert!(!shown("books.set_setting", &write));
assert!(shown("books.import_settings", &write));
}
#[test]
fn a_tool_that_takes_confirmed_posts_only_when_confirmed() {
// Exactly the tools whose arguments have `confirmed`.
@@ -263,7 +364,7 @@ mod tests {
.and_then(|p| p.as_object())
.is_some_and(|p| p.contains_key("confirmed"));
let (_, needs) = TABLE.iter().find(|(n, _)| *n == tool.name).unwrap();
assert_eq!(takes, *needs == C, "{}", tool.name);
assert_eq!(takes, matches!(*needs, C | ImportSettings), "{}", tool.name);
let takes_kind = tool
.input_schema
.get("properties")
+99 -28
View File
@@ -78,6 +78,9 @@ pub struct BooksServer {
/// The files the books made for agents, waiting to be fetched (books
/// served from the data root; over stdio a file comes back inline).
exports: Option<ledger_store::exports::Exports>,
/// The books' backups (books served from the data root; none over
/// stdio).
backups: Option<Arc<crate::backup::Backups>>,
}
/// Where a book's own files live around its data dir.
@@ -146,7 +149,8 @@ impl Layout {
/// whether or not anyone still waits for its answer.
/// ```
pub struct SharedStore {
store: tokio::sync::Mutex<Store>,
/// None once let go (`let_go`): its books were replaced by a restore.
store: tokio::sync::Mutex<Option<Store>>,
stopped: std::sync::atomic::AtomicBool,
/// Closing: no new request runs (INVARIANT 3).
closing: std::sync::atomic::AtomicBool,
@@ -160,6 +164,25 @@ pub struct SharedStore {
}
impl SharedStore {
/// Run `f` on the store under its lock, for a backup (backup.rs):
/// blocking. It reads only, and runs even while the books close (a
/// last backup), never after the server stopped.
pub fn with_store<R>(&self, f: impl FnOnce(&Store) -> R) -> Result<R, String> {
if self.stopped() {
return Err(STOPPED.to_string());
}
let store = self.store.blocking_lock();
let store = store.as_ref().ok_or(STOPPED)?;
Ok(f(store))
}
/// Run `f` on the store if no one holds it now (books just opened):
/// never waits, so never blocks a runtime's thread.
pub fn try_with_store<R>(&self, f: impl FnOnce(&Store) -> R) -> Option<R> {
let store = self.store.try_lock().ok()?;
store.as_ref().map(f)
}
/// An attachment's name, type and bytes (hash-verified), for a
/// download: blocking (the store's lock, then the file).
/// None when there is no such attachment.
@@ -168,6 +191,7 @@ impl SharedStore {
return Err(STOPPED.to_string());
}
let store = self.store.blocking_lock();
let store = store.as_ref().ok_or(STOPPED)?;
let Some(meta) = store
.list_attachments(None)
.map_err(|e| e.to_string())?
@@ -189,7 +213,7 @@ impl SharedStore {
pub fn new(store: Store) -> Arc<Self> {
Arc::new(Self {
currency: store.base_currency(),
store: tokio::sync::Mutex::new(store),
store: tokio::sync::Mutex::new(Some(store)),
stopped: std::sync::atomic::AtomicBool::new(false),
closing: std::sync::atomic::AtomicBool::new(false),
running: std::sync::atomic::AtomicUsize::new(0),
@@ -220,6 +244,15 @@ impl SharedStore {
self.stopped.load(std::sync::atomic::Ordering::SeqCst)
}
/// Let go of the books for good (a restore replaces them): close as
/// [`Self::close`], stop, and drop the store, so its writer lock is
/// released while sessions still hold this.
pub async fn let_go(&self) {
self.close().await;
self.stop();
self.store.lock().await.take();
}
/// Stop taking requests (INVARIANT 1): after an internal error, or an
/// access record that couldn't be written (access.rs INVARIANT 3).
pub(crate) fn stop(&self) {
@@ -275,7 +308,8 @@ const VAULT_IN_HISTORY: &str = "the TIN vault can't store anything here yet: the
/// task is suspended. Dropped by a panic, it stops the server
/// (SharedStore INVARIANT 1).
pub struct StoreGuard<'a> {
guard: tokio::sync::MutexGuard<'a, Store>,
/// Holds a store (checked when it was taken).
guard: tokio::sync::MutexGuard<'a, Option<Store>>,
shared: &'a SharedStore,
/// The journal's head when the lock was taken.
before: (u64, String),
@@ -289,7 +323,10 @@ impl Drop for StoreGuard<'_> {
}
// A write while it was held: the head after it, for the call's
// access record (taken under the same lock, so it is this call's).
let after = self.guard.head();
let Some(store) = self.guard.as_ref() else {
return;
};
let after = store.head();
if after != self.before {
WROTE.with(|w| *w.borrow_mut() = Some(after));
}
@@ -299,13 +336,13 @@ impl Drop for StoreGuard<'_> {
impl std::ops::Deref for StoreGuard<'_> {
type Target = Store;
fn deref(&self) -> &Store {
&self.guard
self.guard.as_ref().expect("a guard holds a store")
}
}
impl std::ops::DerefMut for StoreGuard<'_> {
fn deref_mut(&mut self) -> &mut Store {
&mut self.guard
self.guard.as_mut().expect("a guard holds a store")
}
}
@@ -598,6 +635,7 @@ impl BooksServer {
access: None,
uploads: None,
exports: None,
backups: None,
}
}
@@ -613,6 +651,12 @@ impl BooksServer {
self
}
/// Back the books up through `backups` (backup.rs).
pub fn with_backups(mut self, backups: Arc<crate::backup::Backups>) -> Self {
self.backups = Some(backups);
self
}
/// Check every call against its caller's token, and record it in
/// `log` (access.rs).
pub fn with_access(mut self, log: Arc<ledger_store::access::AccessLog>) -> Self {
@@ -701,7 +745,11 @@ impl BooksServer {
/// Every tool the server serves.
pub(crate) fn tool_router() -> rmcp::handler::server::router::tool::ToolRouter<Self> {
Self::core_router() + Self::kept_router() + Self::access_router() + Self::files_router()
Self::core_router()
+ Self::kept_router()
+ Self::access_router()
+ Self::files_router()
+ Self::backup_router()
}
/// The settings in force for this request (reloaded at its start, in
@@ -1052,21 +1100,25 @@ impl BooksServer {
return Err(STOPPED.to_string());
}
let guard = self.store.store.blocking_lock();
// A request that panicked while this one waited stopped it.
// A request that panicked while this one waited stopped it; books
// let go hold no store.
if self.store.stopped() {
return Err(STOPPED.to_string());
}
let Some(store) = guard.as_ref() else {
return Err(STOPPED.to_string());
};
// Books that keep their own settings: take any change (crate::
// settings INVARIANT 1).
let timezone = {
let mut live = self.live();
live.sync(guard.ledger().settings(), self.store.currency.code());
live.sync(store.ledger().settings(), self.store.currency.code());
live.current().timezone.clone()
};
// "Today" from here on is the zone in force now (a change another
// session made while this request waited included).
TIMEZONE.with(|z| *z.borrow_mut() = timezone);
let before = guard.head();
let before = store.head();
Ok(StoreGuard {
guard,
shared: &self.store,
@@ -2523,6 +2575,8 @@ fn lines_out(lines: &[ledger_core::StatementLine]) -> Vec<StatementLineOut> {
#[path = "tools/access.rs"]
mod access_tools;
#[path = "tools/backup.rs"]
mod backup_tools;
#[path = "tools/files.rs"]
mod files;
#[path = "tools/kept.rs"]
@@ -6982,7 +7036,7 @@ impl BooksServer {
#[tool(
name = "books.purge_tin",
title = "Purge a TIN from the vault",
description = "Delete a party's full TIN and W-9 file from the TIN vault (after the retention period, at the payee's request, or to replace a wrong one), with the reason. Without confirmed=true it previews what the vault holds and changes nothing. Confirmed, the records are deleted for good and the books journal it: the party's TIN facts (party.tin_set: purged says when and why) and W-9 facts (party.w9_set: kept_at says so) keep the last four and drop the vault record. Works without the vault key. The next CPA package then lacks that TIN (and says so).",
description = "Delete a party's full TIN and W-9 file from the TIN vault (after the retention period, at the payee's request, or to replace a wrong one), with the reason. Without confirmed=true it previews what the vault holds and changes nothing. Confirmed, the records are deleted from the vault (backups made before keep their copies) and the books journal it: the party's TIN facts (party.tin_set: purged says when and why) and W-9 facts (party.w9_set: kept_at says so) keep the last four and drop the vault record. Works without the vault key. The next CPA package then lacks that TIN (and says so).",
annotations(
read_only_hint = false,
destructive_hint = true,
@@ -7048,7 +7102,7 @@ impl BooksServer {
.or_else(|| party.tin.as_ref().map(|t| t.last4.clone())),
journaled: false,
note: format!(
"Confirming deletes {} for good{}. Make sure the CPA has what they need first.",
"Confirming deletes {}{}; backups made before keep their copies. Make sure the CPA has what they need first.",
match (held_tin, held_w9) {
(true, true) => "the TIN and the W-9 file",
(true, false) => "the TIN",
@@ -8831,8 +8885,11 @@ impl ServerHandler for BooksServer {
.unwrap_or_default();
tools.retain(|t| crate::scopes::shown(&t.name, &scopes));
} else {
// Books served over stdio keep no access log: not offered.
tools.retain(|t| t.name != "books.access_log");
// Books served over stdio keep no access log and aren't backed
// up here: not offered.
tools.retain(|t| {
!["books.access_log", "books.backup", "books.back_up_now"].contains(&&*t.name)
});
}
let hints = context
.protocol_version()
@@ -8852,7 +8909,9 @@ impl ServerHandler for BooksServer {
// the books' own when they keep them (never waiting for the books
// here; a request holding them syncs them when it locks), else the
// files'.
if let Ok(store) = self.store.store.try_lock() {
if let Ok(store) = self.store.store.try_lock()
&& let Some(store) = store.as_ref()
{
self.live()
.sync(store.ledger().settings(), self.store.currency.code());
}
@@ -9373,7 +9432,7 @@ mod tests {
const D: Hints = (Some(false), Some(true), None);
const WI: Hints = (Some(false), Some(false), Some(true));
const DI: Hints = (Some(false), Some(true), Some(true));
let table: [(&str, Hints); 109] = [
let table: [(&str, Hints); 111] = [
("books.handbook", R),
("books.start", R),
("books.set_setting", DI),
@@ -9483,6 +9542,8 @@ mod tests {
("books.profit_and_loss_by_class", R),
("books.export_cpa_package", W),
("books.access_log", R),
("books.backup", R),
("books.back_up_now", WI),
];
let tools = BooksServer::tool_router().list_all();
assert_eq!(tools.len(), table.len());
@@ -9514,11 +9575,12 @@ mod tests {
for word in ["requiresuserinteraction", "requires_user_interaction"] {
assert!(!json.contains(word), "{name} asks for user interaction");
}
// The books are a closed world: no tool reaches beyond them.
// The books are a closed world: no tool reaches beyond them,
// but the one that pushes a backup to the owner's targets.
assert_eq!(
a.open_world_hint,
Some(false),
"{name} has no openWorldHint false"
Some(*name == "books.back_up_now"),
"{name}: openWorldHint"
);
}
// docs/mcp-interface.md names the 28 destructive tools.
@@ -9581,15 +9643,24 @@ mod tests {
}
assert!(n > 0, "{krate}: no error templates found");
}
// The files tool's refusals (and description): its sentences.
for line in include_str!("tools/files.rs").lines() {
let line = line.trim_start();
if line.starts_with("//") {
continue;
}
for (i, text) in line.split('"').enumerate() {
if i % 2 == 1 && text.len() > 30 && text.contains(' ') {
out.push(("tools/files.rs text".into(), text.to_string()));
// The files and backup tools' refusals (and descriptions), and the
// backups' own words: their sentences. (What git itself says of a
// failed backup reaches the human as git said it: not ours.)
for (name, src) in [
("tools/files.rs", include_str!("tools/files.rs")),
("tools/backup.rs", include_str!("tools/backup.rs")),
("backup.rs", include_str!("backup.rs")),
] {
let src = src.split("#[cfg(test)]").next().unwrap_or(src);
for line in src.lines() {
let line = line.trim_start();
if line.starts_with("//") {
continue;
}
for (i, text) in line.split('"').enumerate() {
if i % 2 == 1 && text.len() > 30 && text.contains(' ') {
out.push((format!("{name} text"), text.to_string()));
}
}
}
}
+127
View File
@@ -0,0 +1,127 @@
//! `books.backup` and `books.back_up_now` (plan 3.7, D-30): the books'
//! backups as they stand, and one made now.
//!
//! ```text
//! GOAL: the agent can tell the human whether the books are backed up,
//! and back them up now when asked.
//! SOURCE: plan 3.7 ("books.backup {status|run} reports it": two tools,
//! one read-only), D-30.
//! INVARIANTS:
//! 1. Only books served from the data root are backed up: over stdio
//! both tools are hidden, and refuse.
//! 2. A status reads the backup record only; a run makes and pushes a
//! backup (backup.rs) and answers the status after it.
//! ```
use super::*;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
#[derive(Debug, Default, Deserialize, JsonSchema)]
pub struct BackupParams {}
#[derive(Debug, Default, Deserialize, JsonSchema)]
pub struct BackUpNowParams {
/// The backup folder's disk is new (or another): begin these books'
/// backup there again. Only when the human says so: a folder that
/// should hold one is otherwise a disk not connected.
#[serde(default)]
pub new_disk: bool,
}
#[derive(Debug, Serialize, JsonSchema)]
pub struct TargetOut {
/// "repository" (the private repository) or "folder".
pub target: String,
/// Where it is.
pub at: String,
/// When it last took a backup (UTC), if ever.
pub last_ok_at: Option<String>,
/// The journal seq it holds, if any.
pub through_seq: Option<u64>,
/// More than a day without the books' latest: tell the human.
pub stale: bool,
/// Why its last backup failed, if it did.
pub last_error: Option<String>,
/// When its backup was last proven restorable (UTC), if ever: each
/// is cloned and checked weekly.
pub last_drill_at: Option<String>,
/// Why that check failed, if it did: tell the human.
pub drill_error: Option<String>,
/// Not backed up there: these books were restored as a copy of others.
pub paused: bool,
}
#[derive(Debug, Serialize, JsonSchema)]
pub struct BackupOut {
/// The books' journal seq now.
pub head_seq: u64,
/// Each backup the settings name.
pub targets: Vec<TargetOut>,
/// What to do, when none is set up.
pub note: Option<String>,
}
impl From<crate::backup::Status> for BackupOut {
fn from(s: crate::backup::Status) -> Self {
BackupOut {
head_seq: s.head_seq,
note: s.note,
targets: s
.targets
.into_iter()
.map(|t| TargetOut {
target: t.target,
at: t.at,
last_ok_at: t.last_ok_at,
through_seq: t.through_seq,
stale: t.stale,
last_error: t.last_error,
last_drill_at: t.last_drill_at,
drill_error: t.drill_error,
paused: t.paused,
})
.collect(),
}
}
}
impl BooksServer {
fn backups(&self) -> Result<&std::sync::Arc<crate::backup::Backups>, String> {
self.backups.as_ref().ok_or_else(|| {
"these books aren't backed up by this server: only books kept in the books' own private folder are".to_string()
})
}
}
#[tool_router(router = backup_router, vis = "pub(super)")]
impl BooksServer {
#[tool(
name = "books.backup",
title = "Backups",
description = "Whether the books are backed up: each backup target the settings name (backup.git_remote, a private repository; backup.folder, a folder on another disk), when it last took the books, through which journal seq, and why it failed if it did. stale: it lacks the latest, and the first try to back that up was more than a day ago: tell the human. The books back themselves up a little after each change, when they open and before they close; books.back_up_now makes one now.",
annotations(read_only_hint = true, open_world_hint = false)
)]
async fn backup(&self, _params: Parameters<BackupParams>) -> Result<Json<BackupOut>, String> {
Ok(Json(self.backups()?.status().await.into()))
}
#[tool(
name = "books.back_up_now",
title = "Back up now",
description = "Back the books up now to every target the settings name, and answer how each went (as books.backup). new_disk: true only when the human says the backup folder's disk is new: the books' backup is then begun there again. Needs books:admin.",
annotations(
read_only_hint = false,
destructive_hint = false,
idempotent_hint = true,
open_world_hint = true
)
)]
async fn back_up_now(
&self,
params: Parameters<BackUpNowParams>,
) -> Result<Json<BackupOut>, String> {
let b = self.backups()?;
Ok(Json(b.run_with("asked", params.0.new_disk).await?.into()))
}
}
+18
View File
@@ -803,6 +803,24 @@ impl BooksServer {
"An open item is due within 30 days, or overdue: raise it with the human.",
));
}
// The books' backups (backup.rs): none set up, or one behind or
// failing.
if let Some(b) = &self.backups {
let st = b.standing(store);
if st.note.is_some() {
next_steps.push(step(
"books.backup",
None,
"No backups are set up: ask the human for a private repository and a folder on another disk, then set backup.git_remote and backup.folder (books.set_setting).",
));
} else if st.needs_attention() {
next_steps.push(step(
"books.backup",
None,
"A backup is behind or failing: books.backup says which and why; tell the human.",
));
}
}
match focus {
"close" => next_steps.push(step(
"books.conventions",
+1 -1
View File
@@ -3383,7 +3383,7 @@ pub struct ReconciliationsOut {
/// Parameters for `books.handbook`.
#[derive(Debug, Default, Deserialize, JsonSchema)]
pub struct HandbookParams {
/// The section to read: contract, entries, drafts-and-reversals, evidence-and-receipts, attachments, files-in-out, reconciliation, fixed-assets, prepaid-and-recurring, revenue-and-payouts, parties-and-tins, 1099, cpa-handoff, settings-and-conventions, viewer, rules-that-change, control-roles, tracing-figures, or "all" for the whole handbook in order. Omit for the index.
/// The section to read: contract, entries, drafts-and-reversals, evidence-and-receipts, attachments, files-in-out, reconciliation, fixed-assets, prepaid-and-recurring, revenue-and-payouts, parties-and-tins, 1099, cpa-handoff, settings-and-conventions, viewer, backups, rules-that-change, control-roles, tracing-figures, or "all" for the whole handbook in order. Omit for the index.
#[serde(default)]
pub section: Option<String>,
}