P2 slice 7: backups and restore (D-30, D-31)
The server backs each book up itself: a per-book git archive, pushed fast-forward only to a git remote and to a folder, when the books open, after a quiet spell, before they close and on request; every commit, push and drill is a record in ops/backup.jsonl. Drills clone each target and verify the chain, the blobs, the vault and the head, weekly and after a failure. `wanderbooks books restore` previews, then restores beside or (with --replace) over, never moving books something holds. The archive refuses anything but a plain bare repository and pins git against the owner's configuration. books.backup and books.back_up_now for the agent. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01T3gXMVt7XJPi6ens1or9Dj
This commit is contained in:
1 parent
5bc0633cdd
commit
019d8d76a3
29 files changed
+6443
-136
No files matched your search
@@ -1,6 +1,6 @@
|
||||
//! The admin socket's messages (`run/admin.sock`, plan 3.1, 3.3, 3.9): how
|
||||
//! the command line asks the books server to list, create and adopt books,
|
||||
//! and to make, list and revoke agents' tokens.
|
||||
//! the command line asks the books server to list, create, adopt and
|
||||
//! restore books, and to make, list and revoke agents' tokens.
|
||||
//!
|
||||
//! ```text
|
||||
//! GOAL: the command line makes and adopts books without ever touching
|
||||
@@ -16,9 +16,10 @@
|
||||
//! that says to restart.
|
||||
//! 3. No answer names the data root or a path in it, nor echoes a path
|
||||
//! the caller gave (which could be the root under another name).
|
||||
//! 4. Adopting is two steps: without `confirmed` the answer is a
|
||||
//! preview, and nothing is written; `confirmed: true` adopts, and
|
||||
//! only books still at the head the preview read (`expect_head`).
|
||||
//! 4. Adopting and restoring are two steps: without `confirmed` the
|
||||
//! answer is a preview, and nothing is written; `confirmed: true`
|
||||
//! adopts (restores), and only books (a backup) still at the head the
|
||||
//! preview read (`expect_head`).
|
||||
//! 5. Nothing in a settings file is dropped silently: entries the books
|
||||
//! don't know are refused unless `skip_unknown`, and its notes and
|
||||
//! comments are in the preview.
|
||||
@@ -97,6 +98,24 @@ pub enum Request {
|
||||
#[serde(default)]
|
||||
skip_unknown: bool,
|
||||
},
|
||||
/// Restore a backup (plan 3.7): as new books, or replacing books
|
||||
/// (which are moved into `legacy/`, kept).
|
||||
Restore {
|
||||
/// The books' id.
|
||||
id: String,
|
||||
/// The backup: a git repository's address, or its folder
|
||||
/// (`<book>.git` in the backup folder).
|
||||
source: String,
|
||||
/// Replace the books called `id`.
|
||||
#[serde(default)]
|
||||
replace: bool,
|
||||
/// Restore (true), or only preview it.
|
||||
#[serde(default)]
|
||||
confirmed: bool,
|
||||
/// The head the preview read: restored only if still so.
|
||||
#[serde(default)]
|
||||
expect_head: Option<(u64, String)>,
|
||||
},
|
||||
/// Make a token for an agent (plan 3.3).
|
||||
TokenCreate {
|
||||
/// The books it opens.
|
||||
@@ -237,6 +256,44 @@ pub struct Adopted {
|
||||
pub problem: Option<String>,
|
||||
}
|
||||
|
||||
/// What a restore would bring in.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct RestorePreview {
|
||||
/// The backup's head, proven.
|
||||
pub head: (u64, String),
|
||||
/// Stored files in it, each re-hashed.
|
||||
pub blobs: u64,
|
||||
/// TIN vault records in it.
|
||||
pub vault_files: u64,
|
||||
/// Whether books of that id exist now.
|
||||
pub exists: bool,
|
||||
/// The id of the books it holds, as they were backed up.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub origin: Option<String>,
|
||||
}
|
||||
|
||||
/// What a restore did.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct Restored {
|
||||
/// The books' id.
|
||||
pub id: String,
|
||||
/// Their head: the backup's.
|
||||
pub head: (u64, String),
|
||||
/// Stored files the books name, re-hashed through the store.
|
||||
pub blobs_verified: u64,
|
||||
/// TIN vault records restored.
|
||||
pub vault_files: u64,
|
||||
/// The `legacy/` folder the replaced books were moved to.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub replaced: Option<String>,
|
||||
/// The id of the books it held, as they were backed up.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub origin: Option<String>,
|
||||
/// Why the restored books didn't open, if they didn't (they are kept).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub problem: Option<String>,
|
||||
}
|
||||
|
||||
/// The server's answer.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case", deny_unknown_fields)]
|
||||
@@ -255,6 +312,10 @@ pub enum Response {
|
||||
Preview(AdoptPreview),
|
||||
/// Books adopted and open.
|
||||
Adopted(Adopted),
|
||||
/// What a restore would bring in.
|
||||
RestorePreview(RestorePreview),
|
||||
/// A backup restored, and open.
|
||||
Restored(Restored),
|
||||
/// A token made: its text, shown this once (INVARIANT 6).
|
||||
TokenCreated {
|
||||
/// The token.
|
||||
@@ -307,7 +368,7 @@ pub fn call_within(
|
||||
// A list changes nothing: its only question is whether it answered.
|
||||
let look = match request {
|
||||
Request::List | Request::TokenList => None,
|
||||
Request::Create { .. } | Request::Adopt { .. } => {
|
||||
Request::Create { .. } | Request::Adopt { .. } | Request::Restore { .. } => {
|
||||
Some("`wanderbooks books list` shows the books there are")
|
||||
}
|
||||
Request::TokenCreate { .. } | Request::TokenRevoke { .. } => {
|
||||
@@ -417,6 +478,24 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_restore_request_is_a_preview_unless_confirmed() {
|
||||
let r: Envelope = serde_json::from_value(
|
||||
json!({"proto": 1, "op": "restore", "id": "acme", "source": "/backups/acme.git"}),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
r.request,
|
||||
Request::Restore {
|
||||
id: "acme".into(),
|
||||
source: "/backups/acme.git".into(),
|
||||
replace: false,
|
||||
confirmed: false,
|
||||
expect_head: None,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_operations_and_fields_are_refused() {
|
||||
for bad in [
|
||||
@@ -455,6 +534,22 @@ mod tests {
|
||||
text: TokenText("wb_00".into()),
|
||||
},
|
||||
Response::Tokens(vec![]),
|
||||
Response::RestorePreview(RestorePreview {
|
||||
head: (3, "ab".into()),
|
||||
blobs: 2,
|
||||
vault_files: 1,
|
||||
exists: true,
|
||||
origin: Some("acme".into()),
|
||||
}),
|
||||
Response::Restored(Restored {
|
||||
id: "acme".into(),
|
||||
head: (3, "ab".into()),
|
||||
blobs_verified: 2,
|
||||
vault_files: 1,
|
||||
replaced: Some("acme-20261019-120000".into()),
|
||||
origin: Some("acme".into()),
|
||||
problem: None,
|
||||
}),
|
||||
] {
|
||||
let text = serde_json::to_string(&r).unwrap();
|
||||
assert_eq!(serde_json::from_str::<Response>(&text).unwrap(), r);
|
||||
|
||||
@@ -0,0 +1,994 @@
|
||||
//! The server's backups of a book (plan 3.7, D-30, D-31): made a while
|
||||
//! after the books are written (when the writing stops), when they open,
|
||||
//! before they close, and when asked; each pushed to the targets the
|
||||
//! book's settings name; each recorded in the book's `ops/backup.jsonl`;
|
||||
//! each target's backup drilled weekly.
|
||||
//!
|
||||
//! ```text
|
||||
//! GOAL: every write reaches the owner's backups soon after it is made,
|
||||
//! and the owner (and agent) can see that it did, or why not.
|
||||
//! SOURCE: plan 3.7 (triggers, "ops/backup.jsonl", books.backup, Needs
|
||||
//! attention for a target more than 24 h behind, drills), D-30 (a
|
||||
//! git repository and a folder), D-31 (plain git).
|
||||
//! INVARIANTS:
|
||||
//! 1. A backup is committed under the store's lock (ledger_store::backup
|
||||
//! INVARIANT 1; its stored files staged before, outside it) and
|
||||
//! pushed outside it: the books are never held while the network is
|
||||
//! slow. One run or drill at a time.
|
||||
//! 2. Every commit, push and drill is a record of `ops/backup.jsonl` (a
|
||||
//! chain log, not the journal: a backup never moves the books' head),
|
||||
//! each target's outcome a record of its own, keyed by the target
|
||||
//! itself (a new folder or address starts with no standing).
|
||||
//! 3. A target is stale when it lacks the books' head and the first try
|
||||
//! to back up what it lacks (the first commit or failure since its
|
||||
//! last good push, or since its first try if it never had one, or
|
||||
//! since the failed commits that kept it from any) is more than
|
||||
//! STALE_AFTER old; its last error is its last push's, or a
|
||||
//! later failed commit's.
|
||||
//! 4. Nothing here names a path of the data root to an agent.
|
||||
//! 5. Every target that holds a backup is drilled (ledger_store::backup::
|
||||
//! drill: cloned, proven whole, its journal the live one's beginning)
|
||||
//! once DRILL_EVERY has passed since its last good drill, or
|
||||
//! RETRY_AFTER since a failed one (at the server's next look for
|
||||
//! drills); a failed drill is said in the standing until one passes.
|
||||
//! 6. A target whose push failed lately (RETRY_AFTER) is left out of a
|
||||
//! backup after a write, and only that one; a folder target's
|
||||
//! repository is made only while the target never took a backup of
|
||||
//! these books (one gone since is a disk not connected), restored
|
||||
//! books counting the backups of the books they were.
|
||||
//! 7. Books restored beside the books they were (under another id) never
|
||||
//! push to the repository those books back up to (it would stop
|
||||
//! theirs): it is paused for them, and said, until another is named.
|
||||
//! 8. A damaged backup record never keeps the books shut: it is set
|
||||
//! aside and a new one begun.
|
||||
//! ```
|
||||
|
||||
use crate::server::SharedStore;
|
||||
use ledger_store::backup::{self, BackupError, Git, Target};
|
||||
use ledger_store::chain::ChainLog;
|
||||
use serde::Serialize;
|
||||
use serde_json::{Map, Value, json};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
/// The book's backup record, in its `ops/`.
|
||||
pub const LOG_FILE: &str = "backup.jsonl";
|
||||
/// The record's genesis text.
|
||||
const GENESIS: &str = "wanderbooks-backup-v1";
|
||||
/// How long a target may lack the books' head before it is stale.
|
||||
pub const STALE_AFTER: chrono::Duration = chrono::Duration::hours(24);
|
||||
/// How often each target's backup is drilled (plan 3.7: weekly).
|
||||
pub const DRILL_EVERY: chrono::Duration = chrono::Duration::days(7);
|
||||
/// How long after a failed run (or drill) the next is tried.
|
||||
pub const RETRY_AFTER: std::time::Duration = std::time::Duration::from_secs(15 * 60);
|
||||
/// The most of an error kept.
|
||||
const ERROR_MAX: usize = 500;
|
||||
|
||||
/// A book's backups.
|
||||
pub struct Backups {
|
||||
book: String,
|
||||
dir: PathBuf,
|
||||
store: Arc<SharedStore>,
|
||||
log: Mutex<ChainLog>,
|
||||
/// One run at a time.
|
||||
running: tokio::sync::Mutex<()>,
|
||||
}
|
||||
|
||||
/// One target's standing (INVARIANT 3).
|
||||
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
|
||||
pub struct TargetStatus {
|
||||
/// "repository" or "folder".
|
||||
pub target: String,
|
||||
/// Where (the repository's address, or "the backup folder").
|
||||
pub at: String,
|
||||
/// When it last took a push, if ever (UTC).
|
||||
pub last_ok_at: Option<String>,
|
||||
/// The journal seq it holds, if any.
|
||||
pub through_seq: Option<u64>,
|
||||
/// Whether it is stale (INVARIANT 3).
|
||||
pub stale: bool,
|
||||
/// Why its last backup failed, if it did.
|
||||
pub last_error: Option<String>,
|
||||
/// When its backup was last drilled (UTC), if ever.
|
||||
pub last_drill_at: Option<String>,
|
||||
/// Why its last drill failed, if it did.
|
||||
pub drill_error: Option<String>,
|
||||
/// Paused for books restored as a copy of others (INVARIANT 7).
|
||||
pub paused: bool,
|
||||
}
|
||||
|
||||
/// The books' backup standing.
|
||||
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
|
||||
pub struct Status {
|
||||
/// The books' head now.
|
||||
pub head_seq: u64,
|
||||
/// Each target the settings name.
|
||||
pub targets: Vec<TargetStatus>,
|
||||
/// None set up: what to do.
|
||||
pub note: Option<String>,
|
||||
}
|
||||
|
||||
/// A target the settings name.
|
||||
#[derive(Debug, Clone)]
|
||||
struct Named {
|
||||
target: Target,
|
||||
/// Where, as an agent is told it.
|
||||
at: String,
|
||||
/// The target itself, as its records name it (INVARIANT 2): a digest
|
||||
/// of where it is (never a path an agent is shown).
|
||||
key: String,
|
||||
}
|
||||
|
||||
/// The targets the book's settings name (D-30).
|
||||
fn targets(store: &ledger_store::Store, book: &str) -> Vec<Named> {
|
||||
use sha2::{Digest, Sha256};
|
||||
let s = store.ledger().settings();
|
||||
let mut out = Vec::new();
|
||||
if let Some(url) = s.backup_git_remote() {
|
||||
out.push((Target::Remote(url.to_string()), url.to_string()));
|
||||
}
|
||||
if let Some(folder) = s.backup_folder() {
|
||||
out.push((
|
||||
Target::Folder(PathBuf::from(folder)),
|
||||
"the backup folder".to_string(),
|
||||
));
|
||||
}
|
||||
out.into_iter()
|
||||
.map(|(target, at)| Named {
|
||||
key: Sha256::digest(target.source(book).as_bytes())[..8]
|
||||
.iter()
|
||||
.map(|b| format!("{b:02x}"))
|
||||
.collect(),
|
||||
target,
|
||||
at,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
impl Backups {
|
||||
/// The backups of the book `book`, kept in `dir`, served from `store`.
|
||||
pub fn open(book: &str, dir: &Path, store: Arc<SharedStore>) -> Result<Self, String> {
|
||||
ledger_store::ops::prepare(dir)
|
||||
.map_err(|e| format!("its backup record can't be kept: {e}"))?;
|
||||
let path = ledger_store::ops::ops_dir(dir).join(LOG_FILE);
|
||||
let (log, carried) = match ChainLog::open(&path, GENESIS) {
|
||||
Ok((log, _)) => (log, Vec::new()),
|
||||
// INVARIANT 8: set aside, begun again, what it knew of whose
|
||||
// backups the targets hold carried into the new one.
|
||||
Err(ledger_store::chain::ChainError::Damaged { .. }) => {
|
||||
let known = known_before_damage(&path);
|
||||
let aside = path.with_extension(format!(
|
||||
"jsonl.damaged-{}",
|
||||
chrono::Utc::now().format("%Y%m%d-%H%M%S")
|
||||
));
|
||||
eprintln!(
|
||||
"wanderbooks server: a backup record was damaged: set aside, begun again"
|
||||
);
|
||||
std::fs::rename(&path, &aside)
|
||||
.map_err(|e| format!("its backup record can't be set aside: {}", e.kind()))?;
|
||||
let log = ChainLog::open(&path, GENESIS)
|
||||
.map_err(|e| format!("its backup record can't be used: {e}"))?
|
||||
.0;
|
||||
(log, known)
|
||||
}
|
||||
Err(e) => return Err(format!("its backup record can't be used: {e}")),
|
||||
};
|
||||
let me = Self {
|
||||
book: book.to_string(),
|
||||
dir: dir.to_path_buf(),
|
||||
store,
|
||||
log: Mutex::new(log),
|
||||
running: tokio::sync::Mutex::new(()),
|
||||
};
|
||||
for fields in carried {
|
||||
me.record(fields);
|
||||
}
|
||||
// Books just restored: whose they were (INVARIANTS 6, 7).
|
||||
let marker = ledger_store::ops::ops_dir(dir).join(ledger_store::root::RESTORED_FROM);
|
||||
if let Ok(origin) = std::fs::read_to_string(&marker) {
|
||||
let origin = Some(origin.trim()).filter(|o| !o.is_empty());
|
||||
let named = me
|
||||
.store
|
||||
.try_with_store(|s| targets(s, book))
|
||||
.ok_or("its backups can't be read yet: the books are busy")?;
|
||||
me.restored(origin, &named);
|
||||
std::fs::remove_file(&marker)
|
||||
.map_err(|e| format!("its backup record can't be kept: {}", e.kind()))?;
|
||||
}
|
||||
Ok(me)
|
||||
}
|
||||
|
||||
fn record(&self, fields: Value) {
|
||||
let Value::Object(mut map) = fields else {
|
||||
return;
|
||||
};
|
||||
// INVARIANT 4: a git or store message may quote a path in the root.
|
||||
if let Some(Value::String(e)) = map.get_mut("error") {
|
||||
*e = hide(e, &self.dir);
|
||||
}
|
||||
let mut log = self.log.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if let Err(e) = log.append(map) {
|
||||
eprintln!("wanderbooks server: the books' backup record couldn't be written: {e}");
|
||||
}
|
||||
}
|
||||
|
||||
fn records(&self) -> Vec<Map<String, Value>> {
|
||||
ledger_store::chain::read(
|
||||
&ledger_store::ops::ops_dir(&self.dir).join(LOG_FILE),
|
||||
GENESIS,
|
||||
)
|
||||
.map(|r| r.records)
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// The targets the settings name now.
|
||||
async fn named(&self) -> Vec<Named> {
|
||||
let (store, book) = (self.store.clone(), self.book.clone());
|
||||
tokio::task::spawn_blocking(move || store.with_store(|s| targets(s, &book)).ok())
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Note that these books were just restored from a backup of the books
|
||||
/// `origin` (INVARIANTS 6, 7), `named` the targets they name: the same
|
||||
/// books (their targets took their backups), or others' (the
|
||||
/// repository those back up to is paused for these).
|
||||
fn restored(&self, origin: Option<&str>, named: &[Named]) {
|
||||
let same = origin == Some(self.book.as_str());
|
||||
let keys = |remote: bool| -> Vec<String> {
|
||||
named
|
||||
.iter()
|
||||
.filter(|n| !remote || matches!(n.target, Target::Remote(_)))
|
||||
.map(|n| n.key.clone())
|
||||
.collect()
|
||||
};
|
||||
if same {
|
||||
self.record(json!({"kind": "restored", "origin": origin, "held": keys(false)}));
|
||||
} else {
|
||||
self.record(json!({"kind": "restored", "origin": origin, "paused": keys(true)}));
|
||||
}
|
||||
}
|
||||
|
||||
/// Wait for any run or drill to end, and start none while the guard
|
||||
/// lives: books being replaced by a restore push nothing more.
|
||||
pub async fn pause(&self) -> tokio::sync::MutexGuard<'_, ()> {
|
||||
self.running.lock().await
|
||||
}
|
||||
|
||||
/// The books' head now.
|
||||
pub async fn head(&self) -> Option<(u64, String)> {
|
||||
let store = self.store.clone();
|
||||
tokio::task::spawn_blocking(move || store.with_store(|s| s.head()).ok())
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
}
|
||||
|
||||
/// Back the books up now (INVARIANTS 1, 2): `why` is recorded (a
|
||||
/// write, open, close, asked). The status after.
|
||||
pub async fn run(&self, why: &str) -> Result<Status, String> {
|
||||
self.run_with(why, false).await
|
||||
}
|
||||
|
||||
/// [`Self::run`]; with `new_disk`, a folder target that took these
|
||||
/// books' backups and holds none now is begun again there (the owner
|
||||
/// says it is a new disk).
|
||||
pub async fn run_with(&self, why: &str, new_disk: bool) -> Result<Status, String> {
|
||||
let _one = self.running.lock().await;
|
||||
if self.named().await.is_empty() {
|
||||
return Ok(self.status().await);
|
||||
}
|
||||
let (store, dir, book) = (self.store.clone(), self.dir.clone(), self.book.clone());
|
||||
let made = tokio::task::spawn_blocking(move || {
|
||||
let git = Git::find()?;
|
||||
// The stored files first, outside the lock: they never change.
|
||||
// Only a head start (a file being written can trip it): the
|
||||
// commit stages them all again.
|
||||
let _ = backup::stage_blobs(&git, &dir);
|
||||
// The commit, under it.
|
||||
store
|
||||
.with_store(|s| -> Result<_, BackupError> {
|
||||
let made = backup::commit(&git, &dir, &book, s.head())?;
|
||||
Ok((made, targets(s, &book), s.head()))
|
||||
})
|
||||
.map_err(BackupError::Damaged)
|
||||
.and_then(|r| r)
|
||||
})
|
||||
.await
|
||||
.unwrap_or(Err(BackupError::Damaged("the backup stopped".into())));
|
||||
let (made, named, head) = match made {
|
||||
Ok(m) => m,
|
||||
Err(e) => {
|
||||
let e = hide(&e.to_string(), &self.dir);
|
||||
self.record(json!({"kind": "commit", "why": why, "ok": false, "error": e}));
|
||||
return Err(e);
|
||||
}
|
||||
};
|
||||
if let Some(m) = &made {
|
||||
self.record(
|
||||
json!({"kind": "commit", "why": why, "ok": true, "commit": m.commit,
|
||||
"head": {"seq": m.head.0, "hash": m.head.1}}),
|
||||
);
|
||||
}
|
||||
// Each target, outside the lock; after a write, one that failed
|
||||
// lately waits out RETRY_AFTER (the others don't wait for it).
|
||||
let records = self.records();
|
||||
let now = chrono::Utc::now();
|
||||
for n in named {
|
||||
if paused(&n.key, &records) || (why == "write" && failed_lately(&n.key, &records, now))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
// Its repository is made only where it never took a backup.
|
||||
let may_create = new_disk || !held_before(&n.key, &records);
|
||||
let (dir, book, t) = (self.dir.clone(), self.book.clone(), n.target.clone());
|
||||
let pushed = tokio::task::spawn_blocking(move || {
|
||||
let git = Git::find()?;
|
||||
backup::push(&git, &dir, &book, &t, may_create)
|
||||
})
|
||||
.await
|
||||
.unwrap_or(Err(BackupError::Damaged("the push stopped".into())));
|
||||
match pushed {
|
||||
Ok(commit) => self.record(
|
||||
json!({"kind": "push", "why": why, "target": n.target.kind(),
|
||||
"at": n.at, "key": n.key, "ok": true, "commit": commit,
|
||||
"head": {"seq": head.0, "hash": head.1}}),
|
||||
),
|
||||
Err(e) => self.record(
|
||||
json!({"kind": "push", "why": why, "target": n.target.kind(),
|
||||
"at": n.at, "key": n.key, "ok": false, "error": e.to_string()}),
|
||||
),
|
||||
}
|
||||
}
|
||||
Ok(self.status().await)
|
||||
}
|
||||
|
||||
/// Whether a drill is due (INVARIANT 5).
|
||||
pub async fn drill_due(&self) -> bool {
|
||||
let records = self.records();
|
||||
let now = chrono::Utc::now();
|
||||
self.named()
|
||||
.await
|
||||
.iter()
|
||||
.any(|n| drill_due(&n.key, &records, now))
|
||||
}
|
||||
|
||||
/// Drill each target whose drill is due (INVARIANT 5), each recorded.
|
||||
pub async fn drill(&self) {
|
||||
let _one = self.running.lock().await;
|
||||
let records = self.records();
|
||||
let now = chrono::Utc::now();
|
||||
for n in self.named().await {
|
||||
if paused(&n.key, &records) || !drill_due(&n.key, &records, now) {
|
||||
continue;
|
||||
}
|
||||
let (dir, book, t) = (self.dir.clone(), self.book.clone(), n.target.clone());
|
||||
let drilled = tokio::task::spawn_blocking(move || {
|
||||
let scratch = ledger_store::ops::ops_dir(&dir).join(format!("drill-{}", t.kind()));
|
||||
let _ = std::fs::remove_dir_all(&scratch);
|
||||
let git = Git::find()?;
|
||||
backup::drill(
|
||||
&git,
|
||||
&t,
|
||||
&book,
|
||||
&dir.join(ledger_store::JOURNAL_FILE),
|
||||
&scratch,
|
||||
)
|
||||
})
|
||||
.await
|
||||
.unwrap_or(Err(BackupError::Damaged("the drill stopped".into())));
|
||||
match drilled {
|
||||
Ok(head) => {
|
||||
self.record(json!({"kind": "drill", "target": n.target.kind(),
|
||||
"at": n.at, "key": n.key, "ok": true, "head": {"seq": head.0, "hash": head.1}}))
|
||||
}
|
||||
Err(e) => self.record(json!({"kind": "drill", "target": n.target.kind(),
|
||||
"at": n.at, "key": n.key, "ok": false, "error": e.to_string()})),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The books' backup standing (INVARIANT 3).
|
||||
pub async fn status(&self) -> Status {
|
||||
let store = self.store.clone();
|
||||
let (book, dir) = (self.book.clone(), self.dir.clone());
|
||||
tokio::task::spawn_blocking(move || store.with_store(|s| standing(s, &book, &dir)).ok())
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
.unwrap_or_else(|| standing_of(&[], &[], 0, chrono::Utc::now()))
|
||||
}
|
||||
|
||||
/// The standing, read from `store` (held by the caller: books.start).
|
||||
pub fn standing(&self, store: &ledger_store::Store) -> Status {
|
||||
standing(store, &self.book, &self.dir)
|
||||
}
|
||||
}
|
||||
|
||||
fn standing(store: &ledger_store::Store, book: &str, dir: &Path) -> Status {
|
||||
let records =
|
||||
ledger_store::chain::read(&ledger_store::ops::ops_dir(dir).join(LOG_FILE), GENESIS)
|
||||
.map(|r| r.records)
|
||||
.unwrap_or_default();
|
||||
standing_of(
|
||||
&targets(store, book),
|
||||
&records,
|
||||
store.head().0,
|
||||
chrono::Utc::now(),
|
||||
)
|
||||
}
|
||||
|
||||
fn standing_of(
|
||||
named: &[Named],
|
||||
records: &[Map<String, Value>],
|
||||
head_seq: u64,
|
||||
now: chrono::DateTime<chrono::Utc>,
|
||||
) -> Status {
|
||||
Status {
|
||||
head_seq,
|
||||
note: named.is_empty().then(|| {
|
||||
"no backups are set up: set backup.git_remote (a private repository) and backup.folder (a folder on another disk) with books.set_setting".to_string()
|
||||
}),
|
||||
targets: named
|
||||
.iter()
|
||||
.map(|n| status_of(n.target.kind(), &n.at, &n.key, records, head_seq, now))
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
impl Status {
|
||||
/// Whether the human should hear of it: none set up, or one stale,
|
||||
/// failing, or failing its drill.
|
||||
pub fn needs_attention(&self) -> bool {
|
||||
self.note.is_some()
|
||||
|| self
|
||||
.targets
|
||||
.iter()
|
||||
.any(|t| t.stale || t.last_error.is_some() || t.drill_error.is_some())
|
||||
}
|
||||
}
|
||||
|
||||
/// `text` with the book's folder `dir`, and the data root above it, named
|
||||
/// in place of their paths (INVARIANT 4); at most ERROR_MAX characters.
|
||||
fn hide(text: &str, dir: &Path) -> String {
|
||||
let mut text = text.to_string();
|
||||
let real = std::fs::canonicalize(dir).unwrap_or_else(|_| dir.to_path_buf());
|
||||
for d in [dir, real.as_path()] {
|
||||
text = text.replace(&d.display().to_string(), "the book's folder");
|
||||
if let Some(root) = d.parent().and_then(Path::parent) {
|
||||
text = text.replace(&root.display().to_string(), ledger_store::root::THE_ROOT);
|
||||
}
|
||||
}
|
||||
text.chars().take(ERROR_MAX).collect()
|
||||
}
|
||||
|
||||
fn is(r: &Map<String, Value>, kind: &str) -> bool {
|
||||
r.get("kind").and_then(Value::as_str) == Some(kind)
|
||||
}
|
||||
|
||||
fn ok(r: &Map<String, Value>) -> bool {
|
||||
r.get("ok") == Some(&Value::Bool(true))
|
||||
}
|
||||
|
||||
fn of(r: &Map<String, Value>, key: &str) -> bool {
|
||||
r.get("key").and_then(Value::as_str) == Some(key)
|
||||
}
|
||||
|
||||
fn ts(r: &Map<String, Value>) -> Option<chrono::DateTime<chrono::Utc>> {
|
||||
r.get("ts")
|
||||
.and_then(Value::as_str)
|
||||
.and_then(|t| chrono::DateTime::parse_from_rfc3339(t).ok())
|
||||
.map(|t| t.with_timezone(&chrono::Utc))
|
||||
}
|
||||
|
||||
fn text(r: &Map<String, Value>, field: &str) -> Option<String> {
|
||||
r.get(field).and_then(Value::as_str).map(str::to_string)
|
||||
}
|
||||
|
||||
/// What a damaged record knew, up to its first line that can't be read at
|
||||
/// all (a line read but out of the chain still counts: what it adds only
|
||||
/// holds more back), of whose backups the
|
||||
/// targets hold: its restored notes, and each target that took a backup
|
||||
/// (as held).
|
||||
fn known_before_damage(path: &Path) -> Vec<Value> {
|
||||
let text = std::fs::read(path).unwrap_or_default();
|
||||
let mut held: Vec<String> = Vec::new();
|
||||
let mut out = Vec::new();
|
||||
for line in text.split(|b| *b == b'\n') {
|
||||
let Ok(Value::Object(r)) = serde_json::from_slice::<Value>(line) else {
|
||||
break;
|
||||
};
|
||||
if is(&r, "push") && ok(&r) {
|
||||
if let Some(k) = text_field(&r, "key")
|
||||
&& !held.contains(&k)
|
||||
{
|
||||
held.push(k);
|
||||
}
|
||||
} else if is(&r, "restored") {
|
||||
let mut r = r.clone();
|
||||
for k in ["seq", "ts", "prev", "hash"] {
|
||||
r.remove(k);
|
||||
}
|
||||
out.push(Value::Object(r));
|
||||
}
|
||||
}
|
||||
if !held.is_empty() {
|
||||
out.push(json!({"kind": "restored", "carried": true, "held": held}));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn text_field(r: &Map<String, Value>, field: &str) -> Option<String> {
|
||||
r.get(field).and_then(Value::as_str).map(str::to_string)
|
||||
}
|
||||
|
||||
/// Whether the target `key` is paused for books restored beside others
|
||||
/// (INVARIANT 7).
|
||||
fn paused(key: &str, records: &[Map<String, Value>]) -> bool {
|
||||
records.iter().any(|r| {
|
||||
is(r, "restored")
|
||||
&& r.get("paused")
|
||||
.and_then(Value::as_array)
|
||||
.is_some_and(|keys| keys.iter().any(|k| k.as_str() == Some(key)))
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether the target `key` took a backup of these books (INVARIANT 6).
|
||||
fn held_before(key: &str, records: &[Map<String, Value>]) -> bool {
|
||||
records.iter().any(|r| {
|
||||
(is(r, "push") && of(r, key) && ok(r))
|
||||
|| (is(r, "restored")
|
||||
&& r.get("held")
|
||||
.and_then(Value::as_array)
|
||||
.is_some_and(|keys| keys.iter().any(|k| k.as_str() == Some(key))))
|
||||
})
|
||||
}
|
||||
|
||||
/// What books restored beside others are told of the repository paused
|
||||
/// for them.
|
||||
pub const PAUSED: &str = "not backed up here: these books were restored as a copy of the books this repository keeps, and backing up here would stop theirs; name a repository of their own (backup.git_remote)";
|
||||
|
||||
/// Whether the target `key`'s last push failed less than RETRY_AFTER ago.
|
||||
fn failed_lately(
|
||||
key: &str,
|
||||
records: &[Map<String, Value>],
|
||||
now: chrono::DateTime<chrono::Utc>,
|
||||
) -> bool {
|
||||
records
|
||||
.iter()
|
||||
.rfind(|r| is(r, "push") && of(r, key))
|
||||
.filter(|r| !ok(r))
|
||||
.and_then(ts)
|
||||
.is_some_and(|at| {
|
||||
now.signed_duration_since(at)
|
||||
< chrono::Duration::from_std(RETRY_AFTER).unwrap_or(chrono::Duration::zero())
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether the target `key`'s drill is due (INVARIANT 5).
|
||||
fn drill_due(
|
||||
key: &str,
|
||||
records: &[Map<String, Value>],
|
||||
now: chrono::DateTime<chrono::Utc>,
|
||||
) -> bool {
|
||||
if !records.iter().any(|r| is(r, "push") && of(r, key) && ok(r)) {
|
||||
return false;
|
||||
}
|
||||
match records.iter().rfind(|r| is(r, "drill") && of(r, key)) {
|
||||
None => true,
|
||||
Some(d) => ts(d).is_none_or(|at| {
|
||||
let wait = if ok(d) {
|
||||
DRILL_EVERY
|
||||
} else {
|
||||
chrono::Duration::from_std(RETRY_AFTER).unwrap_or(DRILL_EVERY)
|
||||
};
|
||||
now.signed_duration_since(at) >= wait
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// A target's standing from the backup record (INVARIANT 3).
|
||||
fn status_of(
|
||||
kind: &str,
|
||||
at: &str,
|
||||
key: &str,
|
||||
records: &[Map<String, Value>],
|
||||
head_seq: u64,
|
||||
now: chrono::DateTime<chrono::Utc>,
|
||||
) -> TargetStatus {
|
||||
let last_ok = records
|
||||
.iter()
|
||||
.rposition(|r| is(r, "push") && of(r, key) && ok(r));
|
||||
let through_seq = last_ok
|
||||
.and_then(|i| records[i].get("head"))
|
||||
.and_then(|h| h.get("seq"))
|
||||
.and_then(Value::as_u64);
|
||||
// Since its last good push (or, never having had one, since its first
|
||||
// try; or, never tried, the failed commits that kept it from one):
|
||||
// this target's tries, and the commits.
|
||||
let from = match last_ok {
|
||||
Some(i) => i + 1,
|
||||
None => records.iter().position(|r| of(r, key)).unwrap_or_else(|| {
|
||||
records
|
||||
.iter()
|
||||
.rposition(|r| !(is(r, "commit") && !ok(r)))
|
||||
.map_or(0, |i| i + 1)
|
||||
}),
|
||||
};
|
||||
let since: Vec<&Map<String, Value>> = records[from..]
|
||||
.iter()
|
||||
.filter(|r| is(r, "commit") || (is(r, "push") && of(r, key)))
|
||||
.collect();
|
||||
let behind = through_seq.is_none_or(|s| s < head_seq);
|
||||
let old = since
|
||||
.first()
|
||||
.and_then(|r| ts(r))
|
||||
.is_some_and(|t| now.signed_duration_since(t) > STALE_AFTER);
|
||||
let last_error = since
|
||||
.iter()
|
||||
.rev()
|
||||
.find(|r| !ok(r) || is(r, "push"))
|
||||
.filter(|r| !ok(r))
|
||||
.and_then(|r| text(r, "error"));
|
||||
let drill = records.iter().rfind(|r| is(r, "drill") && of(r, key));
|
||||
let paused = paused(key, records);
|
||||
TargetStatus {
|
||||
target: kind.to_string(),
|
||||
at: at.to_string(),
|
||||
last_ok_at: last_ok.and_then(|i| text(&records[i], "ts")),
|
||||
through_seq,
|
||||
stale: behind && old && head_seq > 0 && !paused,
|
||||
last_error: if paused {
|
||||
Some(PAUSED.to_string())
|
||||
} else {
|
||||
last_error
|
||||
},
|
||||
last_drill_at: drill.and_then(|r| text(r, "ts")),
|
||||
drill_error: drill.filter(|r| !ok(r)).and_then(|r| text(r, "error")),
|
||||
paused,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn no_error_names_the_root() {
|
||||
let dir = Path::new("/home/u/.local/share/wanderbooks/books/acme");
|
||||
assert_eq!(
|
||||
hide(
|
||||
"fatal: '/home/u/.local/share/wanderbooks/books/acme/archive.git' and /home/u/.local/share/wanderbooks/run",
|
||||
dir
|
||||
),
|
||||
format!(
|
||||
"fatal: 'the book's folder/archive.git' and {}/run",
|
||||
ledger_store::root::THE_ROOT
|
||||
)
|
||||
);
|
||||
assert_eq!(hide(&"x".repeat(2000), dir).len(), ERROR_MAX);
|
||||
}
|
||||
|
||||
/// A record of `kind` for target `key`, `ago` before `now`.
|
||||
fn rec(
|
||||
now: chrono::DateTime<chrono::Utc>,
|
||||
kind: &str,
|
||||
key: &str,
|
||||
ok: bool,
|
||||
seq: u64,
|
||||
ago: chrono::Duration,
|
||||
) -> Map<String, Value> {
|
||||
let mut m = Map::new();
|
||||
m.insert("kind".into(), json!(kind));
|
||||
if kind != "commit" {
|
||||
m.insert("key".into(), json!(key));
|
||||
}
|
||||
m.insert("ok".into(), json!(ok));
|
||||
m.insert("ts".into(), json!((now - ago).to_rfc3339()));
|
||||
if ok {
|
||||
m.insert("head".into(), json!({"seq": seq, "hash": "h"}));
|
||||
} else {
|
||||
m.insert("error".into(), json!(format!("{kind} failed")));
|
||||
}
|
||||
m
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_target_is_stale_only_when_behind_since_a_day() {
|
||||
let now = chrono::Utc::now();
|
||||
let r = |kind, ok, seq, ago| rec(now, kind, "k", ok, seq, ago);
|
||||
let st = |records: &[Map<String, Value>], head| {
|
||||
status_of("folder", "f", "k", records, head, now)
|
||||
};
|
||||
let day = chrono::Duration::hours(24);
|
||||
let hour = chrono::Duration::hours(1);
|
||||
// Nothing tried yet: not stale.
|
||||
assert!(!st(&[], 0).stale);
|
||||
assert!(!st(&[], 4).stale);
|
||||
// Never pushed, the first try a day ago: stale; an hour ago: not.
|
||||
assert!(
|
||||
st(
|
||||
&[
|
||||
r("commit", true, 3, day + hour),
|
||||
r("push", false, 0, day + hour)
|
||||
],
|
||||
3
|
||||
)
|
||||
.stale
|
||||
);
|
||||
assert!(!st(&[r("commit", true, 3, hour), r("push", false, 0, hour)], 3).stale);
|
||||
// Pushed long ago, and quiet since: a new write isn't stale at once.
|
||||
let quiet = [r("commit", true, 3, day * 3), r("push", true, 3, day * 3)];
|
||||
assert!(!st(&quiet, 4).stale);
|
||||
// ... nor while it holds the head.
|
||||
assert!(!st(&quiet, 3).stale);
|
||||
// Behind since a commit a day old, its pushes failing: stale.
|
||||
let failing = [
|
||||
r("push", true, 3, day * 3),
|
||||
r("commit", true, 4, day + hour),
|
||||
r("push", false, 0, day),
|
||||
r("push", false, 0, hour),
|
||||
];
|
||||
let t = st(&failing, 4);
|
||||
assert!(t.stale);
|
||||
assert_eq!(
|
||||
(t.through_seq, t.last_error.as_deref()),
|
||||
(Some(3), Some("push failed"))
|
||||
);
|
||||
// A good push ends the error.
|
||||
let mended = [
|
||||
&failing[..],
|
||||
&[r("push", true, 4, chrono::Duration::zero())],
|
||||
]
|
||||
.concat();
|
||||
let t = st(&mended, 4);
|
||||
assert_eq!(
|
||||
(t.stale, t.last_error, t.through_seq),
|
||||
(false, None, Some(4))
|
||||
);
|
||||
// A failed commit is every target's error, until a good push.
|
||||
let t = st(&[r("push", true, 3, day), r("commit", false, 0, hour)], 4);
|
||||
assert_eq!(t.last_error.as_deref(), Some("commit failed"));
|
||||
// A good commit after a failed push doesn't hide the failure.
|
||||
let t = st(&[r("push", false, 0, hour), r("commit", true, 4, hour)], 4);
|
||||
assert_eq!(t.last_error.as_deref(), Some("push failed"));
|
||||
// Another target's records are not its: a new folder starts clean.
|
||||
let other = [
|
||||
rec(now, "push", "other", true, 3, hour),
|
||||
rec(now, "push", "other", false, 0, hour),
|
||||
];
|
||||
let t = st(&other, 3);
|
||||
assert_eq!(
|
||||
(t.through_seq, t.last_error, t.last_ok_at),
|
||||
(None, None, None)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_target_failing_lately_is_left_out_of_a_write_alone() {
|
||||
let now = chrono::Utc::now();
|
||||
let retry = chrono::Duration::from_std(RETRY_AFTER).unwrap();
|
||||
let second = chrono::Duration::seconds(1);
|
||||
let failed = |ago| rec(now, "push", "k", false, 0, ago);
|
||||
assert!(failed_lately("k", &[failed(retry - second)], now));
|
||||
assert!(!failed_lately("k", &[failed(retry)], now));
|
||||
assert!(!failed_lately(
|
||||
"k",
|
||||
&[failed(second), rec(now, "push", "k", true, 3, second)],
|
||||
now
|
||||
));
|
||||
assert!(!failed_lately("other", &[failed(second)], now));
|
||||
assert!(!failed_lately("k", &[], now));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn commits_failing_from_the_start_are_every_targets_trouble() {
|
||||
let now = chrono::Utc::now();
|
||||
let day = chrono::Duration::hours(24);
|
||||
let hour = chrono::Duration::hours(1);
|
||||
// git missing (say) from the first run on: never a push of its own.
|
||||
let failing = [
|
||||
rec(now, "commit", "", false, 0, day + hour),
|
||||
rec(now, "commit", "", false, 0, hour),
|
||||
];
|
||||
let t = status_of("folder", "f", "k", &failing, 3, now);
|
||||
assert_eq!(t.last_error.as_deref(), Some("commit failed"));
|
||||
assert!(t.stale, "{t:?}");
|
||||
// Failing an hour only: said, not yet stale.
|
||||
let t = status_of("folder", "f", "k", &failing[1..], 3, now);
|
||||
assert_eq!(
|
||||
(t.last_error.as_deref(), t.stale),
|
||||
(Some("commit failed"), false)
|
||||
);
|
||||
// A commit that worked since (its push still to come): no error.
|
||||
let mended = [
|
||||
&failing[..],
|
||||
&[rec(now, "commit", "", true, 3, chrono::Duration::zero())],
|
||||
]
|
||||
.concat();
|
||||
assert_eq!(
|
||||
status_of("folder", "f", "k", &mended, 3, now).last_error,
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restored_books_know_which_backups_were_theirs() {
|
||||
let now = chrono::Utc::now();
|
||||
let restored = |field: &str, keys: &[&str]| {
|
||||
let mut m = Map::new();
|
||||
m.insert("kind".into(), json!("restored"));
|
||||
m.insert(field.into(), json!(keys));
|
||||
m.insert("ts".into(), json!(now.to_rfc3339()));
|
||||
m
|
||||
};
|
||||
// The same books: their targets took their backups.
|
||||
let same = [restored("held", &["k"])];
|
||||
assert!(held_before("k", &same));
|
||||
assert!(!held_before("new", &same));
|
||||
assert!(!paused("k", &same));
|
||||
// Others' (restored beside them): their repository is paused, said.
|
||||
let beside = [restored("paused", &["r"])];
|
||||
assert!(paused("r", &beside));
|
||||
assert!(!paused("k", &beside));
|
||||
assert!(!held_before("r", &beside));
|
||||
let t = status_of("repository", "x", "r", &beside, 9, now);
|
||||
assert_eq!(t.last_error.as_deref(), Some(PAUSED));
|
||||
assert!(!t.stale && t.paused);
|
||||
// A push of its own counts as held, too.
|
||||
assert!(held_before(
|
||||
"k",
|
||||
&[rec(now, "push", "k", true, 3, chrono::Duration::zero())]
|
||||
));
|
||||
assert!(!held_before(
|
||||
"k",
|
||||
&[rec(now, "push", "k", false, 0, chrono::Duration::zero())]
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_target_named_later_starts_its_own_clock() {
|
||||
let now = chrono::Utc::now();
|
||||
let day = chrono::Duration::hours(24);
|
||||
let hour = chrono::Duration::hours(1);
|
||||
// Commits for a week, to another target; then this one's first try.
|
||||
let records = [
|
||||
rec(now, "commit", "", true, 2, day * 7),
|
||||
rec(now, "push", "old", true, 2, day * 7),
|
||||
rec(now, "commit", "", true, 3, day * 2),
|
||||
rec(now, "push", "k", false, 0, hour),
|
||||
];
|
||||
let t = status_of("folder", "f", "k", &records, 3, now);
|
||||
assert!(!t.stale, "{t:?}");
|
||||
assert_eq!(t.last_error.as_deref(), Some("push failed"));
|
||||
// Holding the head: never stale; one behind since long: stale.
|
||||
let held = [
|
||||
rec(now, "push", "k", true, 3, day * 3),
|
||||
rec(now, "push", "k", false, 0, day * 2),
|
||||
];
|
||||
assert!(!status_of("folder", "f", "k", &held, 3, now).stale);
|
||||
assert!(status_of("folder", "f", "k", &held, 4, now).stale);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drills_are_due_weekly_after_a_pass_and_soon_after_a_failure() {
|
||||
let now = chrono::Utc::now();
|
||||
let r = |kind, ok, ago| rec(now, kind, "k", ok, 3, ago);
|
||||
let hour = chrono::Duration::hours(1);
|
||||
let week = DRILL_EVERY;
|
||||
let retry = chrono::Duration::from_std(RETRY_AFTER).unwrap();
|
||||
// Nothing backed up there: nothing to drill.
|
||||
assert!(!drill_due("k", &[], now));
|
||||
assert!(!drill_due("k", &[r("push", false, hour)], now));
|
||||
// Backed up, never drilled: due.
|
||||
let pushed = r("push", true, hour);
|
||||
assert!(drill_due("k", std::slice::from_ref(&pushed), now));
|
||||
// Passed: due a week after.
|
||||
assert!(!drill_due(
|
||||
"k",
|
||||
&[pushed.clone(), r("drill", true, week - hour)],
|
||||
now
|
||||
));
|
||||
assert!(drill_due(
|
||||
"k",
|
||||
&[pushed.clone(), r("drill", true, week)],
|
||||
now
|
||||
));
|
||||
// Failed: tried again RETRY_AFTER after.
|
||||
let t = drill_due(
|
||||
"k",
|
||||
&[
|
||||
pushed.clone(),
|
||||
r("drill", false, retry - chrono::Duration::seconds(1)),
|
||||
],
|
||||
now,
|
||||
);
|
||||
assert!(!t);
|
||||
assert!(drill_due(
|
||||
"k",
|
||||
&[pushed.clone(), r("drill", false, retry)],
|
||||
now
|
||||
));
|
||||
// Another target's drill isn't its, nor its push.
|
||||
assert!(drill_due(
|
||||
"k",
|
||||
&[pushed.clone(), rec(now, "drill", "o", true, 3, hour)],
|
||||
now
|
||||
));
|
||||
assert!(!drill_due(
|
||||
"k",
|
||||
&[rec(now, "push", "o", true, 3, hour)],
|
||||
now
|
||||
));
|
||||
// A failed drill is said until one passes.
|
||||
let t = status_of(
|
||||
"folder",
|
||||
"f",
|
||||
"k",
|
||||
&[pushed.clone(), r("drill", false, hour)],
|
||||
3,
|
||||
now,
|
||||
);
|
||||
assert_eq!(t.drill_error.as_deref(), Some("drill failed"));
|
||||
assert!(t.last_drill_at.is_some());
|
||||
let t = status_of(
|
||||
"folder",
|
||||
"f",
|
||||
"k",
|
||||
&[pushed, r("drill", false, hour), r("drill", true, hour)],
|
||||
3,
|
||||
now,
|
||||
);
|
||||
assert_eq!(t.drill_error, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn each_kind_of_trouble_needs_attention_alone() {
|
||||
let clean = TargetStatus {
|
||||
target: "folder".into(),
|
||||
at: "f".into(),
|
||||
last_ok_at: Some("t".into()),
|
||||
through_seq: Some(3),
|
||||
stale: false,
|
||||
last_error: None,
|
||||
last_drill_at: None,
|
||||
drill_error: None,
|
||||
paused: false,
|
||||
};
|
||||
let status = |t: TargetStatus| Status {
|
||||
head_seq: 3,
|
||||
targets: vec![t],
|
||||
note: None,
|
||||
};
|
||||
assert!(!status(clean.clone()).needs_attention());
|
||||
assert!(
|
||||
status(TargetStatus {
|
||||
stale: true,
|
||||
..clean.clone()
|
||||
})
|
||||
.needs_attention()
|
||||
);
|
||||
assert!(
|
||||
status(TargetStatus {
|
||||
last_error: Some("e".into()),
|
||||
..clean.clone()
|
||||
})
|
||||
.needs_attention()
|
||||
);
|
||||
assert!(
|
||||
status(TargetStatus {
|
||||
drill_error: Some("e".into()),
|
||||
..clean.clone()
|
||||
})
|
||||
.needs_attention()
|
||||
);
|
||||
let none = Status {
|
||||
head_seq: 3,
|
||||
targets: vec![],
|
||||
note: Some("n".into()),
|
||||
};
|
||||
assert!(none.needs_attention());
|
||||
}
|
||||
}
|
||||
@@ -81,6 +81,7 @@ pub const SECTIONS: &[(&str, &str)] = &[
|
||||
include_str!("../handbook/settings-and-conventions.md"),
|
||||
),
|
||||
("viewer", include_str!("../handbook/viewer.md")),
|
||||
("backups", include_str!("../handbook/backups.md")),
|
||||
(
|
||||
"rules-that-change",
|
||||
include_str!("../handbook/rules-that-change.md"),
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
|
||||
pub mod access;
|
||||
pub mod admin;
|
||||
pub mod backup;
|
||||
mod handbook;
|
||||
#[cfg(unix)]
|
||||
pub mod local;
|
||||
|
||||
@@ -75,7 +75,7 @@ use std::os::fd::AsRawFd;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex, RwLock};
|
||||
use std::time::Duration;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tokio_util::task::TaskTracker;
|
||||
|
||||
@@ -103,6 +103,18 @@ pub const REFUSED_HEADER: &str = "x-wanderbooks-refused";
|
||||
|
||||
/// How often each book's lapsed uploads are swept.
|
||||
const SWEEP_EVERY: Duration = Duration::from_secs(10);
|
||||
/// How long the books stay unwritten before they are backed up.
|
||||
const BACKUP_QUIET: Duration = Duration::from_secs(30);
|
||||
/// How often the books are looked at for a backup due.
|
||||
const BACKUP_LOOK: Duration = Duration::from_secs(2);
|
||||
/// How often each book is looked at for a drill due.
|
||||
const DRILL_LOOK: Duration = Duration::from_secs(3600);
|
||||
/// How long after books are first seen open their first drill is looked
|
||||
/// for (a drill that fell due while they were closed): long enough that a
|
||||
/// backup on opening comes first.
|
||||
const DRILL_FIRST: Duration = Duration::from_secs(60);
|
||||
/// The most the last backup at a close may take.
|
||||
const BACKUP_AT_CLOSE: Duration = Duration::from_secs(60);
|
||||
|
||||
type Service = StreamableHttpService<BooksServer, LocalSessionManager>;
|
||||
|
||||
@@ -122,6 +134,8 @@ enum Book {
|
||||
uploads: ledger_store::uploads::Uploads,
|
||||
/// The files it made for agents (`/b/{id}/x/{export}`).
|
||||
exports: ledger_store::exports::Exports,
|
||||
/// Its backups (backup.rs).
|
||||
backups: Arc<crate::backup::Backups>,
|
||||
},
|
||||
/// Not opened, with why (INVARIANT 3).
|
||||
Failed(String),
|
||||
@@ -160,6 +174,14 @@ struct Books {
|
||||
/// Which token opened each MCP session: a session is used only with
|
||||
/// the token that opened it (INVARIANT 8).
|
||||
session_tokens: Mutex<BTreeMap<String, String>>,
|
||||
/// How long the books stay unwritten before a backup is made
|
||||
/// (`--backup-quiet`, for tests; else BACKUP_QUIET).
|
||||
backup_quiet: Duration,
|
||||
/// How long after books are first seen open a drill is first looked
|
||||
/// for (`--drill-first`, for tests; else DRILL_FIRST).
|
||||
drill_first: Duration,
|
||||
/// Held through a restore: one at a time.
|
||||
restoring: Mutex<()>,
|
||||
}
|
||||
|
||||
impl Books {
|
||||
@@ -284,7 +306,17 @@ impl Books {
|
||||
// INVARIANT 2): a viewer can show its presence from the start.
|
||||
ledger_store::ops::prepare(store.data_dir()).map_err(|e| self.hide(e.to_string()))?;
|
||||
let live = Arc::new(Mutex::new(live));
|
||||
let book_dir = store.data_dir().to_path_buf();
|
||||
let book_id = book_dir
|
||||
.file_name()
|
||||
.map(|n| n.to_string_lossy().into_owned())
|
||||
.unwrap_or_default();
|
||||
let store = SharedStore::new(store);
|
||||
let backups = Arc::new(
|
||||
crate::backup::Backups::open(&book_id, &book_dir, store.clone())
|
||||
.map_err(|e| self.hide(e))?,
|
||||
);
|
||||
let server_backups = backups.clone();
|
||||
let log = access.clone();
|
||||
let shared = store.clone();
|
||||
let config = StreamableHttpServerConfig::default()
|
||||
@@ -301,7 +333,8 @@ impl Books {
|
||||
.with_layout(layout.clone())
|
||||
.with_access(log.clone())
|
||||
.with_uploads(server_uploads.clone())
|
||||
.with_exports(server_exports.clone()),
|
||||
.with_exports(server_exports.clone())
|
||||
.with_backups(server_backups.clone()),
|
||||
)
|
||||
},
|
||||
Arc::new(sessions()),
|
||||
@@ -313,6 +346,7 @@ impl Books {
|
||||
access,
|
||||
uploads,
|
||||
exports,
|
||||
backups,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -449,6 +483,13 @@ impl Books {
|
||||
Err(e) => refused(e),
|
||||
}
|
||||
}
|
||||
Request::Restore {
|
||||
id,
|
||||
source,
|
||||
replace,
|
||||
confirmed,
|
||||
expect_head,
|
||||
} => self.restore(id, source, replace, confirmed, expect_head),
|
||||
Request::Adopt {
|
||||
id,
|
||||
source,
|
||||
@@ -589,6 +630,119 @@ impl Books {
|
||||
}
|
||||
}
|
||||
|
||||
impl Books {
|
||||
/// Restore a backup (admin INVARIANT 4; ledger_store::root INVARIANT
|
||||
/// 7): a preview, or the books restored and opened. Books it replaces
|
||||
/// are let go first, and opened again if the restore fails.
|
||||
fn restore(
|
||||
&self,
|
||||
id: String,
|
||||
source: String,
|
||||
replace: bool,
|
||||
confirmed: bool,
|
||||
expect_head: Option<(u64, String)>,
|
||||
) -> Response {
|
||||
// admin INVARIANT 3: never the root, nor the address given.
|
||||
let said = |text: String| {
|
||||
let text = if source.chars().count() > 1 {
|
||||
text.replace(&source, "the backup given")
|
||||
} else {
|
||||
text
|
||||
};
|
||||
Response::Refused(self.hide(text))
|
||||
};
|
||||
let id = match BookId::parse(&id) {
|
||||
Ok(id) => id,
|
||||
Err(e) => return said(e.to_string()),
|
||||
};
|
||||
let _one = self.restoring.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let git = match ledger_store::backup::Git::find() {
|
||||
Ok(git) => git,
|
||||
Err(e) => return said(e.to_string()),
|
||||
};
|
||||
// A preview only reports (whether books of that id exist among
|
||||
// what it says): the restore itself is refused over books, or
|
||||
// replacing none.
|
||||
if !confirmed {
|
||||
return match self.root.restore_preview(&git, &source, &id) {
|
||||
Ok(p) => Response::RestorePreview(admin::RestorePreview {
|
||||
head: p.head,
|
||||
blobs: p.blobs,
|
||||
vault_files: p.vault_files,
|
||||
exists: p.exists,
|
||||
origin: p.origin,
|
||||
}),
|
||||
Err(e) => said(e.to_string()),
|
||||
};
|
||||
}
|
||||
let Some(expect_head) = expect_head else {
|
||||
return said("a restore is confirmed against the preview's head: preview first".into());
|
||||
};
|
||||
let held = self.held().contains_key(id.as_str());
|
||||
if replace && !held {
|
||||
return said(format!(
|
||||
"there are no books called {:?} to replace: nothing was restored",
|
||||
id.as_str()
|
||||
));
|
||||
}
|
||||
if !replace && held {
|
||||
return said(format!(
|
||||
"there are already books called {:?}: restore them under another id (--id), or replace them (--replace; they are kept in legacy/)",
|
||||
id.as_str()
|
||||
));
|
||||
}
|
||||
// The books replaced are let go (their writer lock released),
|
||||
// their backups idle until the restore is done: none of theirs
|
||||
// reaches a target after the backup is read.
|
||||
let old = if replace {
|
||||
self.held
|
||||
.write()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.remove(id.as_str())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let old_backups = match &old {
|
||||
Some(Book::Open { backups, .. }) => Some(backups.clone()),
|
||||
_ => None,
|
||||
};
|
||||
let rt = tokio::runtime::Handle::current();
|
||||
// Its requests end first (a books.back_up_now among them waits for
|
||||
// a run of the loop's), then its last run.
|
||||
if let Some(Book::Open { store, .. }) = &old {
|
||||
rt.block_on(store.let_go());
|
||||
}
|
||||
let _paused = old_backups.as_ref().map(|b| rt.block_on(b.pause()));
|
||||
match self.root.restore(&git, &source, &id, &expect_head, replace) {
|
||||
Ok(r) => {
|
||||
// Restored: kept whether or not it opens; its backups told
|
||||
// whose books these were (backup.rs INVARIANTS 6, 7).
|
||||
let book = self.open(&id);
|
||||
let problem = book.problem();
|
||||
self.hold(&id, book);
|
||||
Response::Restored(admin::Restored {
|
||||
id: r.id.to_string(),
|
||||
head: r.head,
|
||||
blobs_verified: r.blobs_verified,
|
||||
vault_files: r.vault_files,
|
||||
replaced: r.replaced,
|
||||
origin: r.origin,
|
||||
problem,
|
||||
})
|
||||
}
|
||||
Err(e) => {
|
||||
// The books it would have replaced, let go above, open
|
||||
// again (only this restore runs, so nothing else holds them).
|
||||
if replace && !self.held().contains_key(id.as_str()) {
|
||||
let book = self.open(&id);
|
||||
self.hold(&id, book);
|
||||
}
|
||||
said(e.to_string())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The settings a settings file moves in at adoption, checked as
|
||||
/// `books.import_settings` checks them: they parse, they apply, and the
|
||||
/// vault key file it names (which the books don't keep) is the one given.
|
||||
@@ -1603,10 +1757,28 @@ pub fn run(
|
||||
args: impl IntoIterator<Item = String>,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let mut root: Option<PathBuf> = None;
|
||||
let mut backup_quiet = BACKUP_QUIET;
|
||||
let mut drill_first = DRILL_FIRST;
|
||||
let mut args = args.into_iter();
|
||||
while let Some(arg) = args.next() {
|
||||
match arg.as_str() {
|
||||
"--root" => root = Some(PathBuf::from(args.next().ok_or("--root needs a folder")?)),
|
||||
// How long the books stay unwritten before a backup (tests).
|
||||
"--backup-quiet" => {
|
||||
let ms: u64 = args
|
||||
.next()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.ok_or("--backup-quiet needs milliseconds")?;
|
||||
backup_quiet = Duration::from_millis(ms);
|
||||
}
|
||||
// How soon after books open a drill is first looked for (tests).
|
||||
"--drill-first" => {
|
||||
let ms: u64 = args
|
||||
.next()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.ok_or("--drill-first needs milliseconds")?;
|
||||
drill_first = Duration::from_millis(ms);
|
||||
}
|
||||
"--help" | "-h" => {
|
||||
print_usage(prog);
|
||||
return Ok(());
|
||||
@@ -1684,6 +1856,9 @@ pub fn run(
|
||||
started: std::time::Instant::now(),
|
||||
principals: RwLock::new(Err(String::new())),
|
||||
session_tokens: Mutex::new(BTreeMap::new()),
|
||||
backup_quiet,
|
||||
drill_first,
|
||||
restoring: Mutex::new(()),
|
||||
});
|
||||
{
|
||||
let loaded = books.load_principals();
|
||||
@@ -1753,6 +1928,10 @@ async fn serve(
|
||||
}
|
||||
let held: Vec<String> = books.held().keys().cloned().collect();
|
||||
eprintln!("wanderbooks server: ready, holding {} books", held.len());
|
||||
// Every open book is backed up when it opens, once its writing has
|
||||
// stopped for a while, and again after a failed run (backup.rs).
|
||||
let (backing, backup_stop) = (books.clone(), stop.clone());
|
||||
let backup_loop = tokio::spawn(async move { keep_backed_up(backing, backup_stop).await });
|
||||
// Every open book's lapsed uploads go within SWEEP_EVERY of lapsing,
|
||||
// whatever else the server is doing (uploads.rs INVARIANT 3).
|
||||
let (sweeping, sweep_stop) = (books.clone(), stop.clone());
|
||||
@@ -1823,6 +2002,22 @@ async fn serve(
|
||||
for store in books.stores() {
|
||||
store.close().await;
|
||||
}
|
||||
// A last backup of what was written since the last one, for a while
|
||||
// at most: the books are closing, nothing more is written. The loop
|
||||
// stops at its next step (`stop`); a run of its still under way
|
||||
// finishes first (each book runs one at a time).
|
||||
drop(backup_loop);
|
||||
let held: Vec<Arc<crate::backup::Backups>> = books
|
||||
.held()
|
||||
.values()
|
||||
.filter_map(|b| match b {
|
||||
Book::Open { backups, .. } => Some(backups.clone()),
|
||||
Book::Failed(_) => None,
|
||||
})
|
||||
.collect();
|
||||
for b in held {
|
||||
let _ = tokio::time::timeout(BACKUP_AT_CLOSE, b.run("close")).await;
|
||||
}
|
||||
// Answers on their way out are sent, for a while: a client sending
|
||||
// a request it never finishes doesn't hold the books open.
|
||||
let answers = async {
|
||||
@@ -1916,10 +2111,288 @@ fn watch_for_close(stops: &Stops) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// What the backup loop knows of one open book.
|
||||
struct Seen {
|
||||
/// Its backups, by identity: books restored over others are new.
|
||||
backups: Arc<crate::backup::Backups>,
|
||||
watch: Watch,
|
||||
}
|
||||
|
||||
/// One open book's backups, as the loop has watched them.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct Watch {
|
||||
/// Its head's seq when last looked at, and when that changed.
|
||||
seq: u64,
|
||||
moved_at: Instant,
|
||||
/// The seq last committed and offered to the targets.
|
||||
backed: Option<u64>,
|
||||
/// When the last run's commit failed, if it did: nothing runs again
|
||||
/// for RETRY_AFTER.
|
||||
failed_at: Option<Instant>,
|
||||
/// When the last run left a target failing, if it did: it is tried
|
||||
/// again RETRY_AFTER on, written to or not.
|
||||
target_failed_at: Option<Instant>,
|
||||
/// When a drill is next looked for.
|
||||
drill_next: Instant,
|
||||
}
|
||||
|
||||
impl Watch {
|
||||
/// Books first seen open now, at `seq`: a drill is first looked for
|
||||
/// `drill_first` from now.
|
||||
fn new(seq: u64, now: Instant, drill_first: Duration) -> Self {
|
||||
Watch {
|
||||
seq,
|
||||
moved_at: now,
|
||||
backed: None,
|
||||
failed_at: None,
|
||||
target_failed_at: None,
|
||||
drill_next: now + drill_first,
|
||||
}
|
||||
}
|
||||
|
||||
/// A run through `seq` came to `outcome` at `now`: None when its
|
||||
/// commit failed, else whether every target holds it.
|
||||
fn ran(&mut self, seq: u64, outcome: Option<bool>, now: Instant) {
|
||||
match outcome {
|
||||
None => self.failed_at = Some(now),
|
||||
Some(all) => {
|
||||
self.backed = Some(seq);
|
||||
self.failed_at = None;
|
||||
self.target_failed_at = (!all).then_some(now);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Why a backup is due now, if one is: the books just seen open; or
|
||||
/// written and quiet for `quiet` since (not yet backed up); or a target
|
||||
/// left failing RETRY_AFTER ago. Nothing within RETRY_AFTER of a failed
|
||||
/// commit.
|
||||
fn backup_due(
|
||||
seen: Option<&Watch>,
|
||||
seq: u64,
|
||||
now: Instant,
|
||||
quiet: Duration,
|
||||
) -> Option<&'static str> {
|
||||
let Some(s) = seen else {
|
||||
return Some("open");
|
||||
};
|
||||
let after =
|
||||
|t: Option<Instant>| t.is_some_and(|f| now.duration_since(f) >= crate::backup::RETRY_AFTER);
|
||||
if s.failed_at.is_some() && !after(s.failed_at) {
|
||||
return None;
|
||||
}
|
||||
let quiet_now = if s.seq == seq {
|
||||
now.duration_since(s.moved_at) >= quiet
|
||||
} else {
|
||||
quiet.is_zero()
|
||||
};
|
||||
if s.backed != Some(seq) && quiet_now {
|
||||
return Some("write");
|
||||
}
|
||||
after(s.target_failed_at).then_some("retry")
|
||||
}
|
||||
|
||||
/// Whether a run's every target holds the books: a target paused for
|
||||
/// these books (restored as a copy of others) is no failure to try again.
|
||||
fn every_target_held(status: &crate::backup::Status) -> bool {
|
||||
status
|
||||
.targets
|
||||
.iter()
|
||||
.all(|t| t.paused || t.last_error.is_none())
|
||||
}
|
||||
|
||||
/// Whether to look for a drill due now (then again DRILL_LOOK on).
|
||||
fn drill_look_due(seen: &Watch, now: Instant) -> bool {
|
||||
now >= seen.drill_next
|
||||
}
|
||||
|
||||
/// Back every open book up (backup.rs) whenever [`backup_due`] says, and
|
||||
/// drill its backups when a drill is due (looked for every DRILL_LOOK).
|
||||
/// Until `stop`.
|
||||
async fn keep_backed_up(books: Arc<Books>, stop: CancellationToken) {
|
||||
let mut seen: BTreeMap<String, Seen> = BTreeMap::new();
|
||||
let mut tick = tokio::time::interval(BACKUP_LOOK);
|
||||
loop {
|
||||
tokio::select! {
|
||||
() = stop.cancelled() => return,
|
||||
_ = tick.tick() => {}
|
||||
}
|
||||
let open: Vec<(String, Arc<crate::backup::Backups>)> = books
|
||||
.held()
|
||||
.iter()
|
||||
.filter_map(|(id, b)| match b {
|
||||
Book::Open { backups, .. } => Some((id.clone(), backups.clone())),
|
||||
Book::Failed(_) => None,
|
||||
})
|
||||
.collect();
|
||||
seen.retain(|id, _| open.iter().any(|(o, _)| o == id));
|
||||
for (id, backups) in open {
|
||||
if stop.is_cancelled() {
|
||||
return;
|
||||
}
|
||||
let Some((seq, _)) = backups.head().await else {
|
||||
continue;
|
||||
};
|
||||
let now = Instant::now();
|
||||
// Books restored over others are seen anew.
|
||||
if seen
|
||||
.get(&id)
|
||||
.is_some_and(|s| !Arc::ptr_eq(&s.backups, &backups))
|
||||
{
|
||||
seen.remove(&id);
|
||||
}
|
||||
let due = backup_due(
|
||||
seen.get(&id).map(|s| &s.watch),
|
||||
seq,
|
||||
now,
|
||||
books.backup_quiet,
|
||||
);
|
||||
let s = &mut seen
|
||||
.entry(id)
|
||||
.or_insert(Seen {
|
||||
backups: backups.clone(),
|
||||
watch: Watch::new(seq, now, books.drill_first),
|
||||
})
|
||||
.watch;
|
||||
if s.seq != seq {
|
||||
s.seq = seq;
|
||||
s.moved_at = now;
|
||||
}
|
||||
if let Some(why) = due {
|
||||
let outcome = backups
|
||||
.run(why)
|
||||
.await
|
||||
.ok()
|
||||
.map(|status| every_target_held(&status));
|
||||
s.ran(seq, outcome, now);
|
||||
}
|
||||
if stop.is_cancelled() {
|
||||
return;
|
||||
}
|
||||
// Drills (backup.rs INVARIANT 5).
|
||||
if drill_look_due(s, now) {
|
||||
s.drill_next = now + DRILL_LOOK;
|
||||
if backups.drill_due().await {
|
||||
backups.drill().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_backup_is_due_on_open_then_once_quiet_after_a_write() {
|
||||
let t0 = Instant::now();
|
||||
let quiet = Duration::from_secs(30);
|
||||
let s = |seq, moved: u64, backed, failed: Option<u64>| Watch {
|
||||
seq,
|
||||
moved_at: t0 + Duration::from_secs(moved),
|
||||
backed,
|
||||
failed_at: failed.map(|f| t0 + Duration::from_secs(f)),
|
||||
target_failed_at: None,
|
||||
drill_next: t0,
|
||||
};
|
||||
let at = |secs: u64| t0 + Duration::from_secs(secs);
|
||||
// First seen: at once.
|
||||
assert_eq!(backup_due(None, 5, t0, quiet), Some("open"));
|
||||
// Backed up through the head: nothing.
|
||||
assert_eq!(
|
||||
backup_due(Some(&s(5, 0, Some(5), None)), 5, at(100), quiet),
|
||||
None
|
||||
);
|
||||
// Written: not until it has been quiet `quiet`.
|
||||
assert_eq!(
|
||||
backup_due(Some(&s(5, 0, Some(5), None)), 6, at(1), quiet),
|
||||
None
|
||||
);
|
||||
assert_eq!(
|
||||
backup_due(Some(&s(6, 10, Some(5), None)), 6, at(39), quiet),
|
||||
None
|
||||
);
|
||||
assert_eq!(
|
||||
backup_due(Some(&s(6, 10, Some(5), None)), 6, at(40), quiet),
|
||||
Some("write")
|
||||
);
|
||||
// No quiet asked for: at once.
|
||||
assert_eq!(
|
||||
backup_due(Some(&s(5, 0, Some(5), None)), 6, at(1), Duration::ZERO),
|
||||
Some("write")
|
||||
);
|
||||
// A failure waits RETRY_AFTER.
|
||||
let retry = crate::backup::RETRY_AFTER.as_secs();
|
||||
let failed = s(6, 10, Some(5), Some(50));
|
||||
assert_eq!(
|
||||
backup_due(Some(&failed), 6, at(50 + retry - 1), quiet),
|
||||
None
|
||||
);
|
||||
assert_eq!(
|
||||
backup_due(Some(&failed), 6, at(50 + retry), quiet),
|
||||
Some("write")
|
||||
);
|
||||
// A drill is first looked for `drill_first` after the books are
|
||||
// seen open, never at once.
|
||||
let first = Duration::from_secs(60);
|
||||
let w = Watch::new(1, t0, first);
|
||||
assert!(!drill_look_due(&w, t0));
|
||||
assert!(!drill_look_due(&w, t0 + first - Duration::from_millis(1)));
|
||||
assert!(drill_look_due(&w, t0 + first));
|
||||
assert_eq!((w.backed, w.failed_at, w.seq), (None, None, 1));
|
||||
// A run's outcome: its commit failed (nothing again for a
|
||||
// while), or made and some target failing (tried again later,
|
||||
// while writes still go to the others), or all well.
|
||||
let mut w = Watch::new(1, t0, first);
|
||||
w.ran(4, None, at(5));
|
||||
assert_eq!(
|
||||
(w.backed, w.failed_at, w.target_failed_at),
|
||||
(None, Some(at(5)), None)
|
||||
);
|
||||
w.ran(4, Some(false), at(9));
|
||||
assert_eq!(
|
||||
(w.backed, w.failed_at, w.target_failed_at),
|
||||
(Some(4), None, Some(at(9)))
|
||||
);
|
||||
w.seq = 5;
|
||||
w.moved_at = at(10);
|
||||
assert_eq!(backup_due(Some(&w), 5, at(10 + 30), quiet), Some("write"));
|
||||
w.backed = Some(5);
|
||||
assert_eq!(backup_due(Some(&w), 5, at(9 + retry - 1), quiet), None);
|
||||
assert_eq!(backup_due(Some(&w), 5, at(9 + retry), quiet), Some("retry"));
|
||||
// A paused target is no failure; another's error is.
|
||||
let target = |paused: bool, error: Option<&str>| crate::backup::TargetStatus {
|
||||
target: "repository".into(),
|
||||
at: "x".into(),
|
||||
last_ok_at: None,
|
||||
through_seq: None,
|
||||
stale: false,
|
||||
last_error: error.map(str::to_string),
|
||||
last_drill_at: None,
|
||||
drill_error: None,
|
||||
paused,
|
||||
};
|
||||
let status = |targets| crate::backup::Status {
|
||||
head_seq: 5,
|
||||
targets,
|
||||
note: None,
|
||||
};
|
||||
assert!(every_target_held(&status(vec![
|
||||
target(true, Some("paused")),
|
||||
target(false, None)
|
||||
])));
|
||||
assert!(!every_target_held(&status(vec![
|
||||
target(false, Some("no")),
|
||||
target(false, None)
|
||||
])));
|
||||
assert!(every_target_held(&status(vec![])));
|
||||
w.ran(5, Some(true), at(9 + retry));
|
||||
assert_eq!((w.failed_at, w.target_failed_at), (None, None));
|
||||
assert_eq!(backup_due(Some(&w), 5, at(9 + retry * 3), quiet), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_socket_pair_is_this_account_at_both_ends() {
|
||||
let (a, b) = std::os::unix::net::UnixStream::pair().unwrap();
|
||||
|
||||
+112
-11
@@ -23,6 +23,11 @@
|
||||
//! with include_tins, export_cpa_package, get_w9) need books:admin,
|
||||
//! and sealing one into the vault (set_tin, set_w9, which can replace
|
||||
//! the one there) needs books:post (review of slice 4).
|
||||
//! 6. Where the books' backups go is books:admin's, as backing up is
|
||||
//! (a backup is the whole book, its vault and key included):
|
||||
//! setting a `backup.*` key, or importing settings that hold one,
|
||||
//! needs books:admin; other settings books:post as before (review of
|
||||
//! slice 7).
|
||||
//! ```
|
||||
|
||||
use ledger_store::auth::Scope;
|
||||
@@ -38,9 +43,17 @@ pub enum Needs {
|
||||
/// books:post for an approval rule (the control that gates posting),
|
||||
/// books:write for any other convention (INVARIANT 4).
|
||||
PostForApprovalRules,
|
||||
/// books:admin for a change to a `backup.*` setting, books:post for
|
||||
/// any other (INVARIANT 6).
|
||||
AdminForBackupSettings,
|
||||
/// As PostWhenConfirmed, and books:admin when the settings imported
|
||||
/// hold a `[backup]` table (INVARIANT 6).
|
||||
ImportSettings,
|
||||
}
|
||||
|
||||
use Needs::{Always, PostForApprovalRules, PostWhenConfirmed};
|
||||
use Needs::{
|
||||
AdminForBackupSettings, Always, ImportSettings, PostForApprovalRules, PostWhenConfirmed,
|
||||
};
|
||||
const R: Needs = Always(Scope::Read);
|
||||
const W: Needs = Always(Scope::Write);
|
||||
const P: Needs = Always(Scope::Post);
|
||||
@@ -49,12 +62,12 @@ const C: Needs = PostWhenConfirmed;
|
||||
const K: Needs = PostForApprovalRules;
|
||||
|
||||
/// Every tool and what it needs (INVARIANT 1).
|
||||
pub const TABLE: [(&str, Needs); 109] = [
|
||||
pub const TABLE: [(&str, Needs); 111] = [
|
||||
("books.handbook", R),
|
||||
("books.start", R),
|
||||
// Settings are committed at once: the settings commit of the plan.
|
||||
("books.set_setting", P),
|
||||
("books.import_settings", C),
|
||||
("books.set_setting", AdminForBackupSettings),
|
||||
("books.import_settings", ImportSettings),
|
||||
("books.export_settings", R),
|
||||
("books.conventions", R),
|
||||
("books.set_convention", K),
|
||||
@@ -166,6 +179,8 @@ pub const TABLE: [(&str, Needs); 109] = [
|
||||
("books.profit_and_loss_by_class", R),
|
||||
("books.export_cpa_package", A),
|
||||
("books.access_log", A),
|
||||
("books.backup", R),
|
||||
("books.back_up_now", A),
|
||||
];
|
||||
|
||||
/// What `tool` needs, given its arguments; None for a tool this server
|
||||
@@ -178,11 +193,38 @@ pub fn needs(
|
||||
Some(match needs {
|
||||
Always(scope) => *scope,
|
||||
PostWhenConfirmed => {
|
||||
let confirmed = args
|
||||
.and_then(|a| a.get("confirmed"))
|
||||
.and_then(serde_json::Value::as_bool)
|
||||
.unwrap_or(false);
|
||||
if confirmed { Scope::Post } else { Scope::Write }
|
||||
if confirmed(args) {
|
||||
Scope::Post
|
||||
} else {
|
||||
Scope::Write
|
||||
}
|
||||
}
|
||||
AdminForBackupSettings => {
|
||||
let backup = args
|
||||
.and_then(|a| a.get("changes"))
|
||||
.and_then(serde_json::Value::as_array)
|
||||
.is_some_and(|changes| {
|
||||
changes.iter().any(|c| {
|
||||
c.get("key")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.is_some_and(|k| k.trim().to_ascii_lowercase().starts_with("backup."))
|
||||
})
|
||||
});
|
||||
if backup { Scope::Admin } else { Scope::Post }
|
||||
}
|
||||
ImportSettings => {
|
||||
let backup = args
|
||||
.and_then(|a| a.get("toml_text"))
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.and_then(|t| t.parse::<toml::Table>().ok())
|
||||
.is_some_and(|t| t.contains_key("backup"));
|
||||
if backup {
|
||||
Scope::Admin
|
||||
} else if confirmed(args) {
|
||||
Scope::Post
|
||||
} else {
|
||||
Scope::Write
|
||||
}
|
||||
}
|
||||
PostForApprovalRules => {
|
||||
let kind = args
|
||||
@@ -197,15 +239,24 @@ pub fn needs(
|
||||
})
|
||||
}
|
||||
|
||||
fn confirmed(args: Option<&serde_json::Map<String, serde_json::Value>>) -> bool {
|
||||
args.and_then(|a| a.get("confirmed"))
|
||||
.and_then(serde_json::Value::as_bool)
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Whether a token with `scopes` may list `tool` (it can call it in some
|
||||
/// form: a tool that posts when confirmed shows to a books:write token,
|
||||
/// which may preview).
|
||||
pub fn shown(tool: &str, scopes: &std::collections::BTreeSet<Scope>) -> bool {
|
||||
match TABLE.iter().find(|(name, _)| *name == tool) {
|
||||
Some((_, Always(scope))) => scopes.contains(scope),
|
||||
Some((_, PostWhenConfirmed | PostForApprovalRules)) => {
|
||||
Some((_, PostWhenConfirmed | PostForApprovalRules | ImportSettings)) => {
|
||||
scopes.contains(&Scope::Write) || scopes.contains(&Scope::Post)
|
||||
}
|
||||
Some((_, AdminForBackupSettings)) => {
|
||||
scopes.contains(&Scope::Post) || scopes.contains(&Scope::Admin)
|
||||
}
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
@@ -253,6 +304,56 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn where_backups_go_is_admins() {
|
||||
let args = |v: serde_json::Value| v.as_object().cloned();
|
||||
let set = |keys: &[&str]| {
|
||||
let changes: Vec<_> = keys
|
||||
.iter()
|
||||
.map(|k| json!({"key": k, "value": "x"}))
|
||||
.collect();
|
||||
needs(
|
||||
"books.set_setting",
|
||||
args(json!({"changes": changes})).as_ref(),
|
||||
)
|
||||
};
|
||||
assert_eq!(set(&["company.name"]), Some(Scope::Post));
|
||||
assert_eq!(set(&["backup.folder"]), Some(Scope::Admin));
|
||||
assert_eq!(
|
||||
set(&["company.name", "backup.git_remote"]),
|
||||
Some(Scope::Admin)
|
||||
);
|
||||
assert_eq!(set(&[" Backup.Folder"]), Some(Scope::Admin));
|
||||
assert_eq!(needs("books.set_setting", None), Some(Scope::Post));
|
||||
let import = |text: &str, confirmed: bool| {
|
||||
needs(
|
||||
"books.import_settings",
|
||||
args(json!({"toml_text": text, "confirmed": confirmed})).as_ref(),
|
||||
)
|
||||
};
|
||||
assert_eq!(
|
||||
import("[company]\nname = \"A\"\n", false),
|
||||
Some(Scope::Write)
|
||||
);
|
||||
assert_eq!(import("[company]\nname = \"A\"\n", true), Some(Scope::Post));
|
||||
assert_eq!(
|
||||
import("[backup]\nfolder = \"/x\"\n", false),
|
||||
Some(Scope::Admin)
|
||||
);
|
||||
assert_eq!(
|
||||
import("[backup]\nfolder = \"/x\"\n", true),
|
||||
Some(Scope::Admin)
|
||||
);
|
||||
// Shown to whoever can set some setting.
|
||||
let post: std::collections::BTreeSet<Scope> = [Scope::Post].into();
|
||||
let write: std::collections::BTreeSet<Scope> = [Scope::Write].into();
|
||||
let admin: std::collections::BTreeSet<Scope> = [Scope::Admin].into();
|
||||
assert!(shown("books.set_setting", &post));
|
||||
assert!(shown("books.set_setting", &admin));
|
||||
assert!(!shown("books.set_setting", &write));
|
||||
assert!(shown("books.import_settings", &write));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_tool_that_takes_confirmed_posts_only_when_confirmed() {
|
||||
// Exactly the tools whose arguments have `confirmed`.
|
||||
@@ -263,7 +364,7 @@ mod tests {
|
||||
.and_then(|p| p.as_object())
|
||||
.is_some_and(|p| p.contains_key("confirmed"));
|
||||
let (_, needs) = TABLE.iter().find(|(n, _)| *n == tool.name).unwrap();
|
||||
assert_eq!(takes, *needs == C, "{}", tool.name);
|
||||
assert_eq!(takes, matches!(*needs, C | ImportSettings), "{}", tool.name);
|
||||
let takes_kind = tool
|
||||
.input_schema
|
||||
.get("properties")
|
||||
|
||||
@@ -78,6 +78,9 @@ pub struct BooksServer {
|
||||
/// The files the books made for agents, waiting to be fetched (books
|
||||
/// served from the data root; over stdio a file comes back inline).
|
||||
exports: Option<ledger_store::exports::Exports>,
|
||||
/// The books' backups (books served from the data root; none over
|
||||
/// stdio).
|
||||
backups: Option<Arc<crate::backup::Backups>>,
|
||||
}
|
||||
|
||||
/// Where a book's own files live around its data dir.
|
||||
@@ -146,7 +149,8 @@ impl Layout {
|
||||
/// whether or not anyone still waits for its answer.
|
||||
/// ```
|
||||
pub struct SharedStore {
|
||||
store: tokio::sync::Mutex<Store>,
|
||||
/// None once let go (`let_go`): its books were replaced by a restore.
|
||||
store: tokio::sync::Mutex<Option<Store>>,
|
||||
stopped: std::sync::atomic::AtomicBool,
|
||||
/// Closing: no new request runs (INVARIANT 3).
|
||||
closing: std::sync::atomic::AtomicBool,
|
||||
@@ -160,6 +164,25 @@ pub struct SharedStore {
|
||||
}
|
||||
|
||||
impl SharedStore {
|
||||
/// Run `f` on the store under its lock, for a backup (backup.rs):
|
||||
/// blocking. It reads only, and runs even while the books close (a
|
||||
/// last backup), never after the server stopped.
|
||||
pub fn with_store<R>(&self, f: impl FnOnce(&Store) -> R) -> Result<R, String> {
|
||||
if self.stopped() {
|
||||
return Err(STOPPED.to_string());
|
||||
}
|
||||
let store = self.store.blocking_lock();
|
||||
let store = store.as_ref().ok_or(STOPPED)?;
|
||||
Ok(f(store))
|
||||
}
|
||||
|
||||
/// Run `f` on the store if no one holds it now (books just opened):
|
||||
/// never waits, so never blocks a runtime's thread.
|
||||
pub fn try_with_store<R>(&self, f: impl FnOnce(&Store) -> R) -> Option<R> {
|
||||
let store = self.store.try_lock().ok()?;
|
||||
store.as_ref().map(f)
|
||||
}
|
||||
|
||||
/// An attachment's name, type and bytes (hash-verified), for a
|
||||
/// download: blocking (the store's lock, then the file).
|
||||
/// None when there is no such attachment.
|
||||
@@ -168,6 +191,7 @@ impl SharedStore {
|
||||
return Err(STOPPED.to_string());
|
||||
}
|
||||
let store = self.store.blocking_lock();
|
||||
let store = store.as_ref().ok_or(STOPPED)?;
|
||||
let Some(meta) = store
|
||||
.list_attachments(None)
|
||||
.map_err(|e| e.to_string())?
|
||||
@@ -189,7 +213,7 @@ impl SharedStore {
|
||||
pub fn new(store: Store) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
currency: store.base_currency(),
|
||||
store: tokio::sync::Mutex::new(store),
|
||||
store: tokio::sync::Mutex::new(Some(store)),
|
||||
stopped: std::sync::atomic::AtomicBool::new(false),
|
||||
closing: std::sync::atomic::AtomicBool::new(false),
|
||||
running: std::sync::atomic::AtomicUsize::new(0),
|
||||
@@ -220,6 +244,15 @@ impl SharedStore {
|
||||
self.stopped.load(std::sync::atomic::Ordering::SeqCst)
|
||||
}
|
||||
|
||||
/// Let go of the books for good (a restore replaces them): close as
|
||||
/// [`Self::close`], stop, and drop the store, so its writer lock is
|
||||
/// released while sessions still hold this.
|
||||
pub async fn let_go(&self) {
|
||||
self.close().await;
|
||||
self.stop();
|
||||
self.store.lock().await.take();
|
||||
}
|
||||
|
||||
/// Stop taking requests (INVARIANT 1): after an internal error, or an
|
||||
/// access record that couldn't be written (access.rs INVARIANT 3).
|
||||
pub(crate) fn stop(&self) {
|
||||
@@ -275,7 +308,8 @@ const VAULT_IN_HISTORY: &str = "the TIN vault can't store anything here yet: the
|
||||
/// task is suspended. Dropped by a panic, it stops the server
|
||||
/// (SharedStore INVARIANT 1).
|
||||
pub struct StoreGuard<'a> {
|
||||
guard: tokio::sync::MutexGuard<'a, Store>,
|
||||
/// Holds a store (checked when it was taken).
|
||||
guard: tokio::sync::MutexGuard<'a, Option<Store>>,
|
||||
shared: &'a SharedStore,
|
||||
/// The journal's head when the lock was taken.
|
||||
before: (u64, String),
|
||||
@@ -289,7 +323,10 @@ impl Drop for StoreGuard<'_> {
|
||||
}
|
||||
// A write while it was held: the head after it, for the call's
|
||||
// access record (taken under the same lock, so it is this call's).
|
||||
let after = self.guard.head();
|
||||
let Some(store) = self.guard.as_ref() else {
|
||||
return;
|
||||
};
|
||||
let after = store.head();
|
||||
if after != self.before {
|
||||
WROTE.with(|w| *w.borrow_mut() = Some(after));
|
||||
}
|
||||
@@ -299,13 +336,13 @@ impl Drop for StoreGuard<'_> {
|
||||
impl std::ops::Deref for StoreGuard<'_> {
|
||||
type Target = Store;
|
||||
fn deref(&self) -> &Store {
|
||||
&self.guard
|
||||
self.guard.as_ref().expect("a guard holds a store")
|
||||
}
|
||||
}
|
||||
|
||||
impl std::ops::DerefMut for StoreGuard<'_> {
|
||||
fn deref_mut(&mut self) -> &mut Store {
|
||||
&mut self.guard
|
||||
self.guard.as_mut().expect("a guard holds a store")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -598,6 +635,7 @@ impl BooksServer {
|
||||
access: None,
|
||||
uploads: None,
|
||||
exports: None,
|
||||
backups: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -613,6 +651,12 @@ impl BooksServer {
|
||||
self
|
||||
}
|
||||
|
||||
/// Back the books up through `backups` (backup.rs).
|
||||
pub fn with_backups(mut self, backups: Arc<crate::backup::Backups>) -> Self {
|
||||
self.backups = Some(backups);
|
||||
self
|
||||
}
|
||||
|
||||
/// Check every call against its caller's token, and record it in
|
||||
/// `log` (access.rs).
|
||||
pub fn with_access(mut self, log: Arc<ledger_store::access::AccessLog>) -> Self {
|
||||
@@ -701,7 +745,11 @@ impl BooksServer {
|
||||
|
||||
/// Every tool the server serves.
|
||||
pub(crate) fn tool_router() -> rmcp::handler::server::router::tool::ToolRouter<Self> {
|
||||
Self::core_router() + Self::kept_router() + Self::access_router() + Self::files_router()
|
||||
Self::core_router()
|
||||
+ Self::kept_router()
|
||||
+ Self::access_router()
|
||||
+ Self::files_router()
|
||||
+ Self::backup_router()
|
||||
}
|
||||
|
||||
/// The settings in force for this request (reloaded at its start, in
|
||||
@@ -1052,21 +1100,25 @@ impl BooksServer {
|
||||
return Err(STOPPED.to_string());
|
||||
}
|
||||
let guard = self.store.store.blocking_lock();
|
||||
// A request that panicked while this one waited stopped it.
|
||||
// A request that panicked while this one waited stopped it; books
|
||||
// let go hold no store.
|
||||
if self.store.stopped() {
|
||||
return Err(STOPPED.to_string());
|
||||
}
|
||||
let Some(store) = guard.as_ref() else {
|
||||
return Err(STOPPED.to_string());
|
||||
};
|
||||
// Books that keep their own settings: take any change (crate::
|
||||
// settings INVARIANT 1).
|
||||
let timezone = {
|
||||
let mut live = self.live();
|
||||
live.sync(guard.ledger().settings(), self.store.currency.code());
|
||||
live.sync(store.ledger().settings(), self.store.currency.code());
|
||||
live.current().timezone.clone()
|
||||
};
|
||||
// "Today" from here on is the zone in force now (a change another
|
||||
// session made while this request waited included).
|
||||
TIMEZONE.with(|z| *z.borrow_mut() = timezone);
|
||||
let before = guard.head();
|
||||
let before = store.head();
|
||||
Ok(StoreGuard {
|
||||
guard,
|
||||
shared: &self.store,
|
||||
@@ -2523,6 +2575,8 @@ fn lines_out(lines: &[ledger_core::StatementLine]) -> Vec<StatementLineOut> {
|
||||
|
||||
#[path = "tools/access.rs"]
|
||||
mod access_tools;
|
||||
#[path = "tools/backup.rs"]
|
||||
mod backup_tools;
|
||||
#[path = "tools/files.rs"]
|
||||
mod files;
|
||||
#[path = "tools/kept.rs"]
|
||||
@@ -6982,7 +7036,7 @@ impl BooksServer {
|
||||
#[tool(
|
||||
name = "books.purge_tin",
|
||||
title = "Purge a TIN from the vault",
|
||||
description = "Delete a party's full TIN and W-9 file from the TIN vault (after the retention period, at the payee's request, or to replace a wrong one), with the reason. Without confirmed=true it previews what the vault holds and changes nothing. Confirmed, the records are deleted for good and the books journal it: the party's TIN facts (party.tin_set: purged says when and why) and W-9 facts (party.w9_set: kept_at says so) keep the last four and drop the vault record. Works without the vault key. The next CPA package then lacks that TIN (and says so).",
|
||||
description = "Delete a party's full TIN and W-9 file from the TIN vault (after the retention period, at the payee's request, or to replace a wrong one), with the reason. Without confirmed=true it previews what the vault holds and changes nothing. Confirmed, the records are deleted from the vault (backups made before keep their copies) and the books journal it: the party's TIN facts (party.tin_set: purged says when and why) and W-9 facts (party.w9_set: kept_at says so) keep the last four and drop the vault record. Works without the vault key. The next CPA package then lacks that TIN (and says so).",
|
||||
annotations(
|
||||
read_only_hint = false,
|
||||
destructive_hint = true,
|
||||
@@ -7048,7 +7102,7 @@ impl BooksServer {
|
||||
.or_else(|| party.tin.as_ref().map(|t| t.last4.clone())),
|
||||
journaled: false,
|
||||
note: format!(
|
||||
"Confirming deletes {} for good{}. Make sure the CPA has what they need first.",
|
||||
"Confirming deletes {}{}; backups made before keep their copies. Make sure the CPA has what they need first.",
|
||||
match (held_tin, held_w9) {
|
||||
(true, true) => "the TIN and the W-9 file",
|
||||
(true, false) => "the TIN",
|
||||
@@ -8831,8 +8885,11 @@ impl ServerHandler for BooksServer {
|
||||
.unwrap_or_default();
|
||||
tools.retain(|t| crate::scopes::shown(&t.name, &scopes));
|
||||
} else {
|
||||
// Books served over stdio keep no access log: not offered.
|
||||
tools.retain(|t| t.name != "books.access_log");
|
||||
// Books served over stdio keep no access log and aren't backed
|
||||
// up here: not offered.
|
||||
tools.retain(|t| {
|
||||
!["books.access_log", "books.backup", "books.back_up_now"].contains(&&*t.name)
|
||||
});
|
||||
}
|
||||
let hints = context
|
||||
.protocol_version()
|
||||
@@ -8852,7 +8909,9 @@ impl ServerHandler for BooksServer {
|
||||
// the books' own when they keep them (never waiting for the books
|
||||
// here; a request holding them syncs them when it locks), else the
|
||||
// files'.
|
||||
if let Ok(store) = self.store.store.try_lock() {
|
||||
if let Ok(store) = self.store.store.try_lock()
|
||||
&& let Some(store) = store.as_ref()
|
||||
{
|
||||
self.live()
|
||||
.sync(store.ledger().settings(), self.store.currency.code());
|
||||
}
|
||||
@@ -9373,7 +9432,7 @@ mod tests {
|
||||
const D: Hints = (Some(false), Some(true), None);
|
||||
const WI: Hints = (Some(false), Some(false), Some(true));
|
||||
const DI: Hints = (Some(false), Some(true), Some(true));
|
||||
let table: [(&str, Hints); 109] = [
|
||||
let table: [(&str, Hints); 111] = [
|
||||
("books.handbook", R),
|
||||
("books.start", R),
|
||||
("books.set_setting", DI),
|
||||
@@ -9483,6 +9542,8 @@ mod tests {
|
||||
("books.profit_and_loss_by_class", R),
|
||||
("books.export_cpa_package", W),
|
||||
("books.access_log", R),
|
||||
("books.backup", R),
|
||||
("books.back_up_now", WI),
|
||||
];
|
||||
let tools = BooksServer::tool_router().list_all();
|
||||
assert_eq!(tools.len(), table.len());
|
||||
@@ -9514,11 +9575,12 @@ mod tests {
|
||||
for word in ["requiresuserinteraction", "requires_user_interaction"] {
|
||||
assert!(!json.contains(word), "{name} asks for user interaction");
|
||||
}
|
||||
// The books are a closed world: no tool reaches beyond them.
|
||||
// The books are a closed world: no tool reaches beyond them,
|
||||
// but the one that pushes a backup to the owner's targets.
|
||||
assert_eq!(
|
||||
a.open_world_hint,
|
||||
Some(false),
|
||||
"{name} has no openWorldHint false"
|
||||
Some(*name == "books.back_up_now"),
|
||||
"{name}: openWorldHint"
|
||||
);
|
||||
}
|
||||
// docs/mcp-interface.md names the 28 destructive tools.
|
||||
@@ -9581,15 +9643,24 @@ mod tests {
|
||||
}
|
||||
assert!(n > 0, "{krate}: no error templates found");
|
||||
}
|
||||
// The files tool's refusals (and description): its sentences.
|
||||
for line in include_str!("tools/files.rs").lines() {
|
||||
let line = line.trim_start();
|
||||
if line.starts_with("//") {
|
||||
continue;
|
||||
}
|
||||
for (i, text) in line.split('"').enumerate() {
|
||||
if i % 2 == 1 && text.len() > 30 && text.contains(' ') {
|
||||
out.push(("tools/files.rs text".into(), text.to_string()));
|
||||
// The files and backup tools' refusals (and descriptions), and the
|
||||
// backups' own words: their sentences. (What git itself says of a
|
||||
// failed backup reaches the human as git said it: not ours.)
|
||||
for (name, src) in [
|
||||
("tools/files.rs", include_str!("tools/files.rs")),
|
||||
("tools/backup.rs", include_str!("tools/backup.rs")),
|
||||
("backup.rs", include_str!("backup.rs")),
|
||||
] {
|
||||
let src = src.split("#[cfg(test)]").next().unwrap_or(src);
|
||||
for line in src.lines() {
|
||||
let line = line.trim_start();
|
||||
if line.starts_with("//") {
|
||||
continue;
|
||||
}
|
||||
for (i, text) in line.split('"').enumerate() {
|
||||
if i % 2 == 1 && text.len() > 30 && text.contains(' ') {
|
||||
out.push((format!("{name} text"), text.to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
//! `books.backup` and `books.back_up_now` (plan 3.7, D-30): the books'
|
||||
//! backups as they stand, and one made now.
|
||||
//!
|
||||
//! ```text
|
||||
//! GOAL: the agent can tell the human whether the books are backed up,
|
||||
//! and back them up now when asked.
|
||||
//! SOURCE: plan 3.7 ("books.backup {status|run} reports it": two tools,
|
||||
//! one read-only), D-30.
|
||||
//! INVARIANTS:
|
||||
//! 1. Only books served from the data root are backed up: over stdio
|
||||
//! both tools are hidden, and refuse.
|
||||
//! 2. A status reads the backup record only; a run makes and pushes a
|
||||
//! backup (backup.rs) and answers the status after it.
|
||||
//! ```
|
||||
|
||||
use super::*;
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Default, Deserialize, JsonSchema)]
|
||||
pub struct BackupParams {}
|
||||
|
||||
#[derive(Debug, Default, Deserialize, JsonSchema)]
|
||||
pub struct BackUpNowParams {
|
||||
/// The backup folder's disk is new (or another): begin these books'
|
||||
/// backup there again. Only when the human says so: a folder that
|
||||
/// should hold one is otherwise a disk not connected.
|
||||
#[serde(default)]
|
||||
pub new_disk: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, JsonSchema)]
|
||||
pub struct TargetOut {
|
||||
/// "repository" (the private repository) or "folder".
|
||||
pub target: String,
|
||||
/// Where it is.
|
||||
pub at: String,
|
||||
/// When it last took a backup (UTC), if ever.
|
||||
pub last_ok_at: Option<String>,
|
||||
/// The journal seq it holds, if any.
|
||||
pub through_seq: Option<u64>,
|
||||
/// More than a day without the books' latest: tell the human.
|
||||
pub stale: bool,
|
||||
/// Why its last backup failed, if it did.
|
||||
pub last_error: Option<String>,
|
||||
/// When its backup was last proven restorable (UTC), if ever: each
|
||||
/// is cloned and checked weekly.
|
||||
pub last_drill_at: Option<String>,
|
||||
/// Why that check failed, if it did: tell the human.
|
||||
pub drill_error: Option<String>,
|
||||
/// Not backed up there: these books were restored as a copy of others.
|
||||
pub paused: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, JsonSchema)]
|
||||
pub struct BackupOut {
|
||||
/// The books' journal seq now.
|
||||
pub head_seq: u64,
|
||||
/// Each backup the settings name.
|
||||
pub targets: Vec<TargetOut>,
|
||||
/// What to do, when none is set up.
|
||||
pub note: Option<String>,
|
||||
}
|
||||
|
||||
impl From<crate::backup::Status> for BackupOut {
|
||||
fn from(s: crate::backup::Status) -> Self {
|
||||
BackupOut {
|
||||
head_seq: s.head_seq,
|
||||
note: s.note,
|
||||
targets: s
|
||||
.targets
|
||||
.into_iter()
|
||||
.map(|t| TargetOut {
|
||||
target: t.target,
|
||||
at: t.at,
|
||||
last_ok_at: t.last_ok_at,
|
||||
through_seq: t.through_seq,
|
||||
stale: t.stale,
|
||||
last_error: t.last_error,
|
||||
last_drill_at: t.last_drill_at,
|
||||
drill_error: t.drill_error,
|
||||
paused: t.paused,
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl BooksServer {
|
||||
fn backups(&self) -> Result<&std::sync::Arc<crate::backup::Backups>, String> {
|
||||
self.backups.as_ref().ok_or_else(|| {
|
||||
"these books aren't backed up by this server: only books kept in the books' own private folder are".to_string()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[tool_router(router = backup_router, vis = "pub(super)")]
|
||||
impl BooksServer {
|
||||
#[tool(
|
||||
name = "books.backup",
|
||||
title = "Backups",
|
||||
description = "Whether the books are backed up: each backup target the settings name (backup.git_remote, a private repository; backup.folder, a folder on another disk), when it last took the books, through which journal seq, and why it failed if it did. stale: it lacks the latest, and the first try to back that up was more than a day ago: tell the human. The books back themselves up a little after each change, when they open and before they close; books.back_up_now makes one now.",
|
||||
annotations(read_only_hint = true, open_world_hint = false)
|
||||
)]
|
||||
async fn backup(&self, _params: Parameters<BackupParams>) -> Result<Json<BackupOut>, String> {
|
||||
Ok(Json(self.backups()?.status().await.into()))
|
||||
}
|
||||
|
||||
#[tool(
|
||||
name = "books.back_up_now",
|
||||
title = "Back up now",
|
||||
description = "Back the books up now to every target the settings name, and answer how each went (as books.backup). new_disk: true only when the human says the backup folder's disk is new: the books' backup is then begun there again. Needs books:admin.",
|
||||
annotations(
|
||||
read_only_hint = false,
|
||||
destructive_hint = false,
|
||||
idempotent_hint = true,
|
||||
open_world_hint = true
|
||||
)
|
||||
)]
|
||||
async fn back_up_now(
|
||||
&self,
|
||||
params: Parameters<BackUpNowParams>,
|
||||
) -> Result<Json<BackupOut>, String> {
|
||||
let b = self.backups()?;
|
||||
Ok(Json(b.run_with("asked", params.0.new_disk).await?.into()))
|
||||
}
|
||||
}
|
||||
@@ -803,6 +803,24 @@ impl BooksServer {
|
||||
"An open item is due within 30 days, or overdue: raise it with the human.",
|
||||
));
|
||||
}
|
||||
// The books' backups (backup.rs): none set up, or one behind or
|
||||
// failing.
|
||||
if let Some(b) = &self.backups {
|
||||
let st = b.standing(store);
|
||||
if st.note.is_some() {
|
||||
next_steps.push(step(
|
||||
"books.backup",
|
||||
None,
|
||||
"No backups are set up: ask the human for a private repository and a folder on another disk, then set backup.git_remote and backup.folder (books.set_setting).",
|
||||
));
|
||||
} else if st.needs_attention() {
|
||||
next_steps.push(step(
|
||||
"books.backup",
|
||||
None,
|
||||
"A backup is behind or failing: books.backup says which and why; tell the human.",
|
||||
));
|
||||
}
|
||||
}
|
||||
match focus {
|
||||
"close" => next_steps.push(step(
|
||||
"books.conventions",
|
||||
|
||||
@@ -3383,7 +3383,7 @@ pub struct ReconciliationsOut {
|
||||
/// Parameters for `books.handbook`.
|
||||
#[derive(Debug, Default, Deserialize, JsonSchema)]
|
||||
pub struct HandbookParams {
|
||||
/// The section to read: contract, entries, drafts-and-reversals, evidence-and-receipts, attachments, files-in-out, reconciliation, fixed-assets, prepaid-and-recurring, revenue-and-payouts, parties-and-tins, 1099, cpa-handoff, settings-and-conventions, viewer, rules-that-change, control-roles, tracing-figures, or "all" for the whole handbook in order. Omit for the index.
|
||||
/// The section to read: contract, entries, drafts-and-reversals, evidence-and-receipts, attachments, files-in-out, reconciliation, fixed-assets, prepaid-and-recurring, revenue-and-payouts, parties-and-tins, 1099, cpa-handoff, settings-and-conventions, viewer, backups, rules-that-change, control-roles, tracing-figures, or "all" for the whole handbook in order. Omit for the index.
|
||||
#[serde(default)]
|
||||
pub section: Option<String>,
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user