ci: shard the weekly mutation runs; no viewer job; six import mutants caught
check / check (push) Canceled after 0s

The first scheduled run (2026-10-19) failed four ways: the viewer's
mutation job called the recipe retired in P2 slice 1 (owner, 2026-10-18:
no mutation runs on the GUI); the core and MCP runs were stopped at
GitHub's six hours; the import run missed six mutants.

- Every mutation recipe takes a shard (k/n, default all of it); the
  workflow runs the core in 8 shards, the store in 4, the importers in 2,
  the MCP server in 16, each its own job with a 350-minute limit.
- Test timeouts are three times each run's own baseline, never under the
  recipe's old fixed value, so slower runners don't report false
  timeouts.
- The mutants-gui job is gone.
- ledger-import: rule_marks_match_their_notes,
  asset_register_xlsx_headings_and_totals_rule and
  xlsx_missing_amounts_are_cells_only_on_ruled_rows catch the six
  (the tax-basis rule marks, a heading's alignment, the totals row's
  blank cells, and a blank amount cell only on ruled rows), each checked
  against its mutant by hand.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01T3gXMVt7XJPi6ens1or9Dj
This commit is contained in:
swaitsandClaude Opus 5.5 committed 2026-10-19 10:00:38 -06:00
1 parent a82791789e
commit 67648473ba
5 files changed
+219 -40

No files matched your search

+54 -27
View File
@@ -1,4 +1,8 @@
# Weekly, and on demand: the slow layers (docs/testing.md).
#
# Mutation testing is split into shards (`just mutants-* k/n`), each its
# own job, so every job ends well inside GitHub's six hours. The viewer has
# no mutation job (owner, 2026-10-18).
name: scheduled
on:
schedule:
@@ -29,54 +33,77 @@ jobs:
- run: just fuzz ${{ matrix.target }} 600
mutants:
runs-on: ubuntu-24.04
timeout-minutes: 350
strategy:
fail-fast: false
matrix:
shard: [0, 1, 2, 3, 4, 5, 6, 7]
steps:
- uses: actions/checkout@v4
- uses: jdx/mise-action@v2
- run: just mutants
- run: just mutants-store
- run: just mutants ${{ matrix.shard }}/8
- if: always()
uses: actions/upload-artifact@v4
with:
name: mutants
path: |
mutants.out
mutants-store/mutants.out
mutants-import:
name: mutants-core-${{ matrix.shard }}
path: mutants.out
mutants-store:
runs-on: ubuntu-24.04
timeout-minutes: 350
strategy:
fail-fast: false
matrix:
shard: [0, 1, 2, 3]
steps:
- uses: actions/checkout@v4
- uses: jdx/mise-action@v2
- run: just mutants-store ${{ matrix.shard }}/4
- if: always()
uses: actions/upload-artifact@v4
with:
name: mutants-store-${{ matrix.shard }}
path: mutants-store/mutants.out
mutants-import:
runs-on: ubuntu-24.04
timeout-minutes: 350
strategy:
fail-fast: false
matrix:
shard: [0, 1]
steps:
- uses: actions/checkout@v4
- uses: jdx/mise-action@v2
- run: just mutants-import ${{ matrix.shard }}/2
- if: always()
uses: actions/upload-artifact@v4
with:
name: mutants-import-${{ matrix.shard }}
path: mutants-import/mutants.out
mutants-config:
runs-on: ubuntu-24.04
timeout-minutes: 350
steps:
- uses: actions/checkout@v4
- uses: jdx/mise-action@v2
- run: just mutants-import
- run: just mutants-config
- if: always()
uses: actions/upload-artifact@v4
with:
name: mutants-import
path: |
mutants-import/mutants.out
mutants-config/mutants.out
name: mutants-config
path: mutants-config/mutants.out
mutants-mcp:
runs-on: ubuntu-24.04
timeout-minutes: 350
strategy:
fail-fast: false
matrix:
shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
steps:
- uses: actions/checkout@v4
- uses: jdx/mise-action@v2
- run: just mutants-mcp
- run: just mutants-mcp ${{ matrix.shard }}/16
- if: always()
uses: actions/upload-artifact@v4
with:
name: mutants-mcp
name: mutants-mcp-${{ matrix.shard }}
path: mutants-mcp/mutants.out
mutants-gui:
runs-on: ubuntu-24.04
timeout-minutes: 350
steps:
- uses: actions/checkout@v4
- uses: jdx/mise-action@v2
- run: sudo apt-get update && sudo apt-get install -y libxkbcommon-dev libgtk-3-dev
- run: just mutants-gui
- if: always()
uses: actions/upload-artifact@v4
with:
name: mutants-gui
path: mutants-gui/mutants.out
+46 -1
View File
@@ -6,7 +6,8 @@
use chrono::NaiveDate;
use ledger_core::{AssetRegister, AssetRegisterRow, TaxBasisRule};
use ledger_import::asset_register::{
layout, missing_line, notes, rule_cited, rule_label, typeset_asset_register,
layout, missing_line, notes, rule_cited, rule_label, rule_mark, template_data,
typeset_asset_register,
};
use ledger_import::report::ReportMeta;
use ledger_import::{PdfError, asset_register_pdf};
@@ -277,6 +278,50 @@ fn rule_words() {
);
}
/// INVARIANT 9: the notes number the rules 1, 2, 3 (the general note
/// first and unnumbered, the built-in gain note 4), each rule's note
/// cites that rule, and the PDF marks each recorded line's rule with its
/// note's number; a line with no rule recorded has no mark.
#[test]
fn rule_marks_match_their_notes() {
use TaxBasisRule::*;
let rules = [Cost, Carryover, PersonalUseConversion];
assert_eq!(rules.map(rule_mark), ["1", "2", "3"]);
let n = notes();
let marks: Vec<&str> = n.iter().map(|n| n.mark.as_str()).collect();
assert_eq!(marks, ["", "1", "2", "3", "4"]);
for rule in rules {
let note = n.iter().find(|n| n.mark == rule_mark(rule)).unwrap();
let cites = format!("{}: ", rule_cited(rule));
assert!(note.text.starts_with(&cites), "{rule:?}: {:?}", note.text);
}
// The template's input: each line its rule's mark, in register order.
let r = sample();
let data = template_data(&r, &meta());
let marks: Vec<&str> = data["lines"]
.as_array()
.unwrap()
.iter()
.map(|l| l["rule_mark"].as_str().unwrap())
.collect();
assert_eq!(marks, ["1", "2", "1", "3", ""]);
// The PDF: the mark right after the rule, the note under its number.
let t = all_text(&r, &meta());
for s in [
"(Ryzen 7 7840U, 64 GB, 4 TB) Cost 1 2,500.00",
"MacBook Air Carryover 2 1,299.00",
"Old monitor Cost 1 400.00",
"Sony A7 IV camera Personal-use conversion 3 1,650.00",
"Standing desk Not recorded 800.00",
"1 Cost: property the company bought",
"2 Carryover (IRC §723): property a member contributes",
"3 Personal-use conversion (Treas. Reg. §1.168(i)-4(b)): property converted",
"4 Built-in gain (loss):",
] {
assert!(t.contains(s), "{s:?} missing from {t:?}");
}
}
/// INVARIANT 4: an empty register renders a "No fixed assets" line, its
/// totals, and the reconciliation.
#[test]
+102
View File
@@ -2828,6 +2828,63 @@ fn xlsx_statement_layout_and_rules() {
assert!(w.contains_key(&1) && w.contains_key(&2), "{w:?}");
}
/// INVARIANT 5's blank cells: a missing amount on a ruled row (a subtotal
/// under amounts, a total) is a blank cell in that row's ruled amount
/// format; on any other row (a line, a subtotal with nothing above it)
/// it writes no cell at all.
#[test]
fn xlsx_missing_amounts_are_cells_only_on_ruled_rows() {
let row = |kind, label: &str, amounts: Vec<Option<Decimal>>| Row {
kind,
label: label.into(),
amounts,
currency: false,
rule_above: false,
};
let st = report::statement(
"Blanks",
"July 2025".into(),
&["Book", "Tax"],
vec![
row(RowKind::Section, "Income", vec![]),
row(RowKind::Line, "Sales", vec![Some(cents(100)), None]),
row(RowKind::Line, "Fees", vec![Some(cents(-5)), None]),
row(
RowKind::Subtotal,
"Total income",
vec![Some(cents(95)), None],
),
row(RowKind::Spacer, "", vec![]),
row(RowKind::Section, "Empty", vec![]),
row(RowKind::Subtotal, "Total empty", vec![Some(cents(0)), None]),
row(RowKind::Spacer, "", vec![]),
row(RowKind::Total, "Net", vec![Some(cents(95)), None]),
],
);
let ruled: Vec<&str> = st
.rows
.iter()
.filter(|r| r.rule_above)
.map(|r| r.label.as_str())
.collect();
assert_eq!(ruled, ["Total income", "Net"]);
let bytes = report_to_xlsx(&XlsxReport::Statement(&st), &meta()).unwrap();
let cells = sheet_cells(&bytes);
let styles = cell_styles(&bytes);
for r in st.rows.iter().filter(|r| !r.amounts.is_empty()) {
let at = row_of(&cells, &r.label, 0).unwrap();
assert!(cells.contains_key(&(at, 1)), "{:?}: its amount", r.label);
assert!(!cells.contains_key(&(at, 2)), "{:?}: no value", r.label);
if r.rule_above {
let blank = styles.get(&(at, 2));
assert!(blank.is_some_and(|s| s.top_border), "{:?}", r.label);
assert_eq!(blank, styles.get(&(at, 1)), "{:?}: as its amount", r.label);
} else {
assert!(!styles.contains_key(&(at, 2)), "{:?}: no cell", r.label);
}
}
}
/// INVARIANT 4 on the general ledger and journal: amount headings
/// right-aligned and the others not, accounts one empty row apart, and the
/// footer one row below (also with nothing to show).
@@ -3217,12 +3274,57 @@ fn asset_register_xlsx_cells() {
assert!(
text_cell(&c, (22, 0)).starts_with("Book value is the fair market value at contribution")
);
for (row, start) in [
(23, "1. Cost: "),
(24, "2. Carryover (IRC §723): "),
(
25,
"3. Personal-use conversion (Treas. Reg. §1.168(i)-4(b)): ",
),
(26, "4. Built-in gain (loss): "),
] {
assert!(text_cell(&c, (row, 0)).starts_with(start), "note {start:?}");
}
let footer = c
.values()
.any(|v| matches!(v, Cell::Text(t) if t.contains("Journal seq 42")));
assert!(footer, "footer present");
}
/// asset_register INVARIANTS 2 and 5 as laid out (crate::xlsx INVARIANTS
/// 4-5): the headings over the amounts and over the life in months
/// right-aligned and the others not, and the totals row ruled across
/// every column after its label in the totals format, blank cells and
/// totals alike; the asset rows above it aren't ruled.
#[test]
fn asset_register_xlsx_headings_and_totals_rule() {
let x = ledger_import::asset_register_xlsx(&asset_register_fixture(), &meta()).unwrap();
let c = sheet_cells(&x);
let styles = cell_styles(&x);
let right_aligned = [4, 5, 6, 7, 8, 10, 11, 12];
for col in 0..14 {
let right = styles[&(5, col)].halign.as_deref() == Some("right");
assert_eq!(right, right_aligned.contains(&col), "heading column {col}");
}
assert_eq!(text_cell(&c, (9, 0)), "Total, assets held");
assert!(!styles[&(9, 0)].top_border, "the label isn't ruled");
let total = &styles[&(9, 4)];
assert!(total.top_border, "{total:?}");
for col in 1..14 {
assert_eq!(
styles.get(&(9, col)),
Some(total),
"totals row column {col}"
);
}
for row in 6..=8 {
for col in 0..14 {
let ruled = styles.get(&(row, col)).is_some_and(|s| s.top_border);
assert!(!ruled, "asset row {row} column {col}");
}
}
}
/// asset_register INVARIANT 4: an empty register says so, still totals,
/// and still reconciles (the books may carry unregistered cost).
#[test]
+2 -2
View File
@@ -30,14 +30,14 @@ promise.
| Benchmarks | a baseline (6) | `ledger-core/benches/books.rs`, `ledger-store/benches/store.rs` | `just bench` | ~2 min | none (no budget, by decision) |
| Mutation | can the tests fail? | `ledger-core` (2026-10-13: 1,011 caught + 13 timeouts of 1,024 viable, 0 missed) | `just mutants` | ~3 h | weekly |
| Mutation (store) | can the tests fail? | `ledger-store` (2026-10-13: 357 caught + 1 timeout of 365 viable, from 82%; three of its seven survivors then killed; four equivalent left: `sort_value`'s two arms, `OpenFlags` `|` vs `^`, the long form of a sweepless close) | `just mutants-store` | ~12 min | weekly |
| Mutation (import) | can the tests fail? | `ledger-import` (2026-10-13: from 91 of 171 (53%) to 208 of 239 viable on the full run; the 31 survivors (XLSX layout, borders, alignment, exact-text grouping, IIF transliteration, tax-summary optional sections, day-equals-month dates) then killed by targeted tests, re-run clean but one: a blank amount cell written unbordered on an unruled row, invisible, so equivalent) | `just mutants-import` | ~1 h | weekly |
| Mutation (import) | can the tests fail? | `ledger-import` (2026-10-13: from 91 of 171 (53%) to 208 of 239 viable on the full run; the 31 survivors (XLSX layout, borders, alignment, exact-text grouping, IIF transliteration, tax-summary optional sections, day-equals-month dates) then killed by targeted tests, re-run clean but one: a blank amount cell written unbordered on an unruled row, invisible, so equivalent; 2026-10-19, the first scheduled run in CI: 375 tested, 327 caught, 42 unviable, 6 missed (the tax-basis rule marks, a heading's alignment, the totals row's blank cells, and that blank cell, after all: a ruled row writes it, an unruled one writes none), all six then caught by `rule_marks_match_their_notes`, `asset_register_xlsx_headings_and_totals_rule` and `xlsx_missing_amounts_are_cells_only_on_ruled_rows`, each checked against its mutant by hand) | `just mutants-import` | ~2 h | weekly |
| Mutation (roles, reload) | can the tests fail? | control roles and settings reload (2026-10-15, targeted: the role paths in `ledger-core` 30 caught of 30 viable; `ledger-store`'s journaling and replay of `role.assigned` 20 of 20; `ledger-mcp` settings, reload, roles, digest 28 of 28) | `cargo mutants -F` on those functions (see the commit) | ~15 min | with the crate's weekly run |
| Mutation (config) | can the tests fail? | `ledger-config` (2026-10-14: 55 caught of 55 viable) | `just mutants-config` | minutes | weekly |
| Mutation (MCP) | can the tests fail? | `ledger-mcp` (2026-10-14: from 197 caught + 5 timeouts of 328 viable (62%) to 314 caught + 2 timeouts of 331; of its 15 survivors, five then killed by targeted tests (the settings file beside the books, year earnings with an open period and with last year's partial close, year 1, a failed flush), one dead guard removed (`profit_and_loss_by_class` skipping placeholders, which never take postings), nine equivalent: `money` rescaling an amount already at two places, the lock wait's deadline instant, `year_earnings` starting the day the year starts either way, the bank import's rehearsal sides (a line's side can't make a draft refuse) and its zero-amount side (zero rows are dropped by the parser), a disposal's zero gain adding zero, and the CPA package index summing debits instead of credits (entries balance)) | `just mutants-mcp` | ~1 h | weekly |
| Mutation (GUI) | can the tests fail? | `ledger-gui` view model and formatting (2026-10-14: 140 caught of 176 viable on the full run, then 92 of 99 on a targeted rerun of the money paths after new tests; of its 7 survivors, one then killed by a test (burn over exactly one month, confirmed by a rerun of `compute_kpis`'s 48 mutants), two dead arms removed (`parties`' bill arms: the ledger tags bill entries with their vendor, which wins), four equivalent: an entry's debits vs its credits (they balance), the three-month burn boundary (both paths give 3), burn over zero covered months (only when the first entry is after the window, which then holds no loss), and `compact`'s sign test at zero (zero prints "$0" unsigned either way)) | retired 2026-10-18 (owner: it crushed the machine; for the viewer, test its two musts directly: it never writes the books, `tests/read_only.rs`, and every figure it shows is the engine's, `tests/view_model.rs`) | | never |
CI: `.github/workflows/check.yml` runs `just check` on every push;
`scheduled.yml` runs proofs, the oracle, fuzzing, and mutation weekly.
`scheduled.yml` runs proofs, the oracle, fuzzing, and mutation weekly. Mutation is split into shards, one job each (`just mutants-mcp 3/16` runs the fourth of sixteen), because GitHub stops a job at six hours and a whole crate takes longer on its runners (2026-10-19, the first scheduled run: the core and MCP runs were stopped at the limit). Test timeouts are three times each run's own baseline, never under the recipe's floor, so a slower runner doesn't report false timeouts. The viewer has no mutation job.
Seeds and failure databases: proptest's `*.proptest-regressions` files
are checked in and replay on every run. The GnuCash oracle's books come
+15 -10
View File
@@ -88,40 +88,45 @@ miri:
# Where cargo-mutants copies and builds the tree for each mutant (several
# GB per job): the user's cache on disk, never /tmp, which is often RAM.
# A mutant only needs some test to fail, so proptest doesn't shrink.
#
# Every mutation recipe takes a shard, "k/n" (default: all of it), so CI
# can spread a crate over parallel jobs. Test timeouts are three times the
# run's own baseline, never under the floor given: a slower machine gets
# longer ones instead of false timeouts.
mutants_tmp := env_var_or_default("XDG_CACHE_HOME", env_var("HOME") + "/.cache") + "/wanderbooks/mutants"
# Mutation testing of the core: do the tests catch planted bugs?
mutants: sweep-test-tmp
mutants shard="0/1": sweep-test-tmp
# Core and store tests (the store's replay tests reach Ledger::replay_*);
# the whole workspace would rebuild the viewer for every mutant.
mkdir -p {{mutants_tmp}}
PROPTEST_MAX_SHRINK_ITERS=0 TMPDIR={{mutants_tmp}} cargo mutants -p ledger-core --exclude crates/ledger-core/src/proofs.rs --test-package ledger-core --test-package ledger-store -j 4 --timeout 120
PROPTEST_MAX_SHRINK_ITERS=0 TMPDIR={{mutants_tmp}} cargo mutants -p ledger-core --exclude crates/ledger-core/src/proofs.rs --test-package ledger-core --test-package ledger-store -j 4 --timeout-multiplier 3 --minimum-test-timeout 120 --shard {{shard}}
@just sweep-test-tmp
# Mutation testing of the store. Fewer random cases per mutant (each needs
# a run that catches it, not the deep sweep).
mutants-store: sweep-test-tmp
mutants-store shard="0/1": sweep-test-tmp
mkdir -p {{mutants_tmp}}
STORE_CASES=64 PROPTEST_MAX_SHRINK_ITERS=0 TMPDIR={{mutants_tmp}} cargo mutants -p ledger-store --test-package ledger-store -j 3 --timeout 300 -o mutants-store
STORE_CASES=64 PROPTEST_MAX_SHRINK_ITERS=0 TMPDIR={{mutants_tmp}} cargo mutants -p ledger-store --test-package ledger-store -j 3 --timeout-multiplier 3 --minimum-test-timeout 300 --shard {{shard}} -o mutants-store
@just sweep-test-tmp
# Mutation testing of the importers and report renderers.
mutants-import: sweep-test-tmp
mutants-import shard="0/1": sweep-test-tmp
mkdir -p {{mutants_tmp}}
PROPTEST_MAX_SHRINK_ITERS=0 TMPDIR={{mutants_tmp}} cargo mutants -p ledger-import --test-package ledger-import -j 4 --timeout 300 -o mutants-import
PROPTEST_MAX_SHRINK_ITERS=0 TMPDIR={{mutants_tmp}} cargo mutants -p ledger-import --test-package ledger-import -j 4 --timeout-multiplier 3 --minimum-test-timeout 300 --shard {{shard}} -o mutants-import
@just sweep-test-tmp
# Mutation testing of the settings parser and the receipt policy.
mutants-config: sweep-test-tmp
mutants-config shard="0/1": sweep-test-tmp
mkdir -p {{mutants_tmp}}
PROPTEST_MAX_SHRINK_ITERS=0 TMPDIR={{mutants_tmp}} cargo mutants -p ledger-config --test-package ledger-config -j 4 --timeout 120 -o mutants-config
PROPTEST_MAX_SHRINK_ITERS=0 TMPDIR={{mutants_tmp}} cargo mutants -p ledger-config --test-package ledger-config -j 4 --timeout-multiplier 3 --minimum-test-timeout 120 --shard {{shard}} -o mutants-config
@just sweep-test-tmp
# Mutation testing of the MCP server (every mutant runs the stdio suites;
# fewer random cases each, as for the store).
mutants-mcp: sweep-test-tmp
mutants-mcp shard="0/1": sweep-test-tmp
mkdir -p {{mutants_tmp}}
MCP_CASES=16 PROPTEST_MAX_SHRINK_ITERS=0 TMPDIR={{mutants_tmp}} cargo mutants -p ledger-mcp --test-package ledger-mcp -j 4 --timeout 600 -o mutants-mcp
MCP_CASES=16 PROPTEST_MAX_SHRINK_ITERS=0 TMPDIR={{mutants_tmp}} cargo mutants -p ledger-mcp --test-package ledger-mcp -j 4 --timeout-multiplier 3 --minimum-test-timeout 600 --shard {{shard}} -o mutants-mcp
@just sweep-test-tmp
# No mutation runs on the viewer (owner, 2026-10-18: they crushed the