store: set aside a torn last journal line on open; server waits for lock

A crash or power loss mid-append can leave a partial last line. The
writer acknowledges an append only after its whole line is synced, so
that line was never acknowledged: opening now saves those bytes to
journal.jsonl.torn.<time>, truncates to the last newline, and journals
journal.tail_repaired (replay and projection ignore it; the GUI shows
it; books.digest reports it in journal_repairs). Damage before the last
newline is still corruption and still refuses to open. Tested at every
byte of the last append.

The MCP server now waits up to ten seconds for the writer lock: a
restarted server (a client reconnecting) raced the exiting one and died
with Locked, which also made mcp_schedules_and_recurring_over_stdio
flaky. The test harness now waits for each server to exit.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01GWVo7x3UHw5kwCUaWxiejX

crates/ledger-store/src/journal.rs

@@ -112,8 +112,11 @@ pub fn sha256_hex(bytes: &[u8]) -> String {
 }
 
 /// Append-only writer. Assigns `seq`, stamps `ts`, chains `prev`, fsyncs
-/// every append. The sequence only advances after a successful fsync, so a
-/// torn tail can never exist in a file written by this writer.
+/// every append. The sequence only advances after a successful fsync, and a
+/// failed write is truncated away, so an append is acknowledged only once
+/// its whole line is on disk. A crash or power loss mid-append can still
+/// leave a partial last line; it was never acknowledged, and
+/// [`repair_torn_tail`] sets it aside when the books are next opened.
 pub struct JournalWriter {
     path: PathBuf,
     file: File,
@@ -224,10 +227,74 @@ pub fn read_and_verify_journal(path: &Path) -> Result<Vec<JournalLine>, StoreErr
     Ok(lines)
 }
 
+/// A torn final journal line, set aside when the books were opened.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct TornTail {
+    /// How many bytes followed the last complete line.
+    pub bytes: u64,
+    /// SHA-256 of those bytes.
+    pub sha256: String,
+    /// Where they were saved, beside the journal.
+    pub saved_as: PathBuf,
+}
+
+/// Set aside a torn final line: bytes after the journal's last newline.
+///
+/// ```text
+/// GOAL: open books whose last append was cut off (power loss, crash).
+/// SOURCE: docs/receipt-store.md (a command is acknowledged only after its
+///   whole line, newline included, is synced); the same rule write-ahead
+///   logs use for a torn tail record (SQLite, PostgreSQL: recovery stops
+///   at the first incomplete record).
+/// SCENARIOS:
+///   1. IF the journal ends with a newline: SHOULD change nothing.
+///   2. IF bytes follow the last newline: SHOULD save them to a file
+///      beside the journal, truncate the journal to the last newline, and
+///      report them
+///      BECAUSE [`JournalWriter::append`] syncs the newline before it
+///      returns, so a line without one was never acknowledged: no one was
+///      told it happened, and dropping it loses nothing.
+///   3. NEVER touch a complete line: damage before the last newline is
+///      corruption, and opening refuses it.
+/// ```
+pub fn repair_torn_tail(path: &Path) -> Result<Option<TornTail>, StoreError> {
+    let raw = std::fs::read(path)?;
+    let keep = raw.iter().rposition(|&b| b == b'\n').map_or(0, |i| i + 1);
+    if keep == raw.len() {
+        return Ok(None);
+    }
+    let tail = &raw[keep..];
+    let stamp = chrono::Utc::now().format("%Y%m%dT%H%M%S%.fZ").to_string();
+    let mut saved_as = path.as_os_str().to_owned();
+    saved_as.push(format!(".torn.{stamp}"));
+    let saved_as = PathBuf::from(saved_as);
+    {
+        let mut f = OpenOptions::new()
+            .write(true)
+            .create_new(true)
+            .open(&saved_as)?;
+        f.write_all(tail)?;
+        f.sync_all()?;
+    }
+    if let Some(dir) = path.parent()
+        && let Ok(d) = File::open(dir)
+    {
+        let _ = d.sync_all();
+    }
+    let file = OpenOptions::new().write(true).open(path)?;
+    file.set_len(keep as u64)?;
+    file.sync_all()?;
+    Ok(Some(TornTail {
+        bytes: tail.len() as u64,
+        sha256: sha256_hex(tail),
+        saved_as,
+    }))
+}
+
 /// Truncate a journal file to its last good line. For recovering *copies*
 /// with a torn tail (e.g. a backup taken mid-write): the projection never
 /// saw the torn line, so dropping it is safe. Returns the last good seq.
-/// The live journal written by [`JournalWriter`] cannot have a torn tail.
+/// The live journal's torn tail is handled on open by [`repair_torn_tail`].
 pub fn truncate_to_last_good_seq(path: &Path) -> Result<u64, StoreError> {
     let raw = std::fs::read(path)?;
     let mut good_len = 0usize;

crates/ledger-store/src/store.rs

@@ -237,6 +237,8 @@ impl Store {
         }
         // The writer lock first: nothing is read until no one else writes.
         let lock = lock_books(data_dir)?;
+        // An append cut off mid-line was never acknowledged: set it aside.
+        let torn = crate::journal::repair_torn_tail(&journal_path)?;
         // Single verified scan; the writer reuses it.
         let lines = read_and_verify_journal(&journal_path)?;
         let journal = JournalWriter::from_lines(&journal_path, &lines)?;
@@ -305,6 +307,23 @@ impl Store {
             base_currency,
         };
         store.catch_up(&lines)?;
+        if let Some(t) = torn {
+            // On the record, so the repair is part of the audit trail.
+            let line = store.append_journal(
+                "journal.tail_repaired",
+                json!({
+                    "bytes": t.bytes,
+                    "sha256": t.sha256,
+                    "saved_as": t.saved_as.file_name().map(|n| n.to_string_lossy().into_owned()),
+                }),
+            )?;
+            apply_line_to_sqlite(&store.conn, &line)?;
+            eprintln!(
+                "wanderbooks: the journal's last line was cut off (an append that never finished); set aside {} bytes in {}",
+                t.bytes,
+                t.saved_as.display()
+            );
+        }
         Ok(store)
     }
 
@@ -414,6 +433,23 @@ impl Store {
         self.base_currency
     }
 
+    /// Every torn journal tail set aside on open, oldest first, as
+    /// (when, bytes, file it was saved to).
+    pub fn tail_repairs(&self) -> Result<Vec<(String, u64, String)>, StoreError> {
+        let lines = crate::journal::read_and_verify_journal(&self.data_dir.join(JOURNAL_FILE))?;
+        Ok(lines
+            .iter()
+            .filter(|l| l.line_type == "journal.tail_repaired")
+            .map(|l| {
+                (
+                    l.ts.clone(),
+                    l.payload["bytes"].as_u64().unwrap_or(0),
+                    l.payload["saved_as"].as_str().unwrap_or("").to_string(),
+                )
+            })
+            .collect())
+    }
+
     /// Data directory.
     pub fn data_dir(&self) -> &Path {
         &self.data_dir
@@ -3340,6 +3376,7 @@ pub(crate) fn replay_line_into_ledger(
                 p.get("class_id").and_then(|v| v.as_u64()).map(|v| v as u32),
             )?;
         }
+        "journal.tail_repaired" => {}
         other => {
             return Err(StoreError::JournalCorrupt(format!(
                 "unknown line type: {other}"
@@ -3545,7 +3582,7 @@ pub(crate) fn apply_line_to_sqlite(
 fn apply_line_rows(conn: &Connection, line: &JournalLine) -> Result<(), StoreError> {
     let p = &line.payload;
     match line.line_type.as_str() {
-        "store.created" => {}
+        "store.created" | "journal.tail_repaired" => {}
         "account.added" => {
             conn.execute(
                 "INSERT INTO accounts(id, code, name, type, parent, placeholder, currency) VALUES (?,?,?,?,?,?,?)",