From 67648473ba8214bd11224a0a6d1a394fa5295568 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Mon, 19 Oct 2026 09:57:55 -0600 Subject: [PATCH] ci: shard the weekly mutation runs; no viewer job; six import mutants caught The first scheduled run (2026-10-19) failed four ways: the viewer's mutation job called the recipe retired in P2 slice 1 (owner, 2026-10-18: no mutation runs on the GUI); the core and MCP runs were stopped at GitHub's six hours; the import run missed six mutants. - Every mutation recipe takes a shard (k/n, default all of it); the workflow runs the core in 8 shards, the store in 4, the importers in 2, the MCP server in 16, each its own job with a 350-minute limit. - Test timeouts are three times each run's own baseline, never under the recipe's old fixed value, so slower runners don't report false timeouts. - The mutants-gui job is gone. - ledger-import: rule_marks_match_their_notes, asset_register_xlsx_headings_and_totals_rule and xlsx_missing_amounts_are_cells_only_on_ruled_rows catch the six (the tax-basis rule marks, a heading's alignment, the totals row's blank cells, and a blank amount cell only on ruled rows), each checked against its mutant by hand. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01T3gXMVt7XJPi6ens1or9Dj --- .github/workflows/scheduled.yml | 81 ++++++++++----- crates/ledger-import/tests/asset_register.rs | 47 ++++++++- crates/ledger-import/tests/spec_import.rs | 102 +++++++++++++++++++ docs/testing.md | 4 +- justfile | 25 +++-- 5 files changed, 219 insertions(+), 40 deletions(-) diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 65b3483..2c71af5 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -1,4 +1,8 @@ # Weekly, and on demand: the slow layers (docs/testing.md). +# +# Mutation testing is split into shards (`just mutants-* k/n`), each its +# own job, so every job ends well inside GitHub's six hours. The viewer has +# no mutation job (owner, 2026-10-18). name: scheduled on: schedule: @@ -29,54 +33,77 @@ jobs: - run: just fuzz ${{ matrix.target }} 600 mutants: runs-on: ubuntu-24.04 + timeout-minutes: 350 + strategy: + fail-fast: false + matrix: + shard: [0, 1, 2, 3, 4, 5, 6, 7] steps: - uses: actions/checkout@v4 - uses: jdx/mise-action@v2 - - run: just mutants - - run: just mutants-store + - run: just mutants ${{ matrix.shard }}/8 - if: always() uses: actions/upload-artifact@v4 with: - name: mutants - path: | - mutants.out - mutants-store/mutants.out - mutants-import: + name: mutants-core-${{ matrix.shard }} + path: mutants.out + mutants-store: runs-on: ubuntu-24.04 + timeout-minutes: 350 + strategy: + fail-fast: false + matrix: + shard: [0, 1, 2, 3] + steps: + - uses: actions/checkout@v4 + - uses: jdx/mise-action@v2 + - run: just mutants-store ${{ matrix.shard }}/4 + - if: always() + uses: actions/upload-artifact@v4 + with: + name: mutants-store-${{ matrix.shard }} + path: mutants-store/mutants.out + mutants-import: + runs-on: ubuntu-24.04 + timeout-minutes: 350 + strategy: + fail-fast: false + matrix: + shard: [0, 1] + steps: + - uses: actions/checkout@v4 + - uses: jdx/mise-action@v2 + - run: just mutants-import ${{ matrix.shard }}/2 + - if: always() + uses: actions/upload-artifact@v4 + with: + name: mutants-import-${{ matrix.shard }} + path: mutants-import/mutants.out + mutants-config: + runs-on: ubuntu-24.04 + timeout-minutes: 350 steps: - uses: actions/checkout@v4 - uses: jdx/mise-action@v2 - - run: just mutants-import - run: just mutants-config - if: always() uses: actions/upload-artifact@v4 with: - name: mutants-import - path: | - mutants-import/mutants.out - mutants-config/mutants.out + name: mutants-config + path: mutants-config/mutants.out mutants-mcp: runs-on: ubuntu-24.04 timeout-minutes: 350 + strategy: + fail-fast: false + matrix: + shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] steps: - uses: actions/checkout@v4 - uses: jdx/mise-action@v2 - - run: just mutants-mcp + - run: just mutants-mcp ${{ matrix.shard }}/16 - if: always() uses: actions/upload-artifact@v4 with: - name: mutants-mcp + name: mutants-mcp-${{ matrix.shard }} path: mutants-mcp/mutants.out - mutants-gui: - runs-on: ubuntu-24.04 - timeout-minutes: 350 - steps: - - uses: actions/checkout@v4 - - uses: jdx/mise-action@v2 - - run: sudo apt-get update && sudo apt-get install -y libxkbcommon-dev libgtk-3-dev - - run: just mutants-gui - - if: always() - uses: actions/upload-artifact@v4 - with: - name: mutants-gui - path: mutants-gui/mutants.out diff --git a/crates/ledger-import/tests/asset_register.rs b/crates/ledger-import/tests/asset_register.rs index 200a60c..017a791 100644 --- a/crates/ledger-import/tests/asset_register.rs +++ b/crates/ledger-import/tests/asset_register.rs @@ -6,7 +6,8 @@ use chrono::NaiveDate; use ledger_core::{AssetRegister, AssetRegisterRow, TaxBasisRule}; use ledger_import::asset_register::{ - layout, missing_line, notes, rule_cited, rule_label, typeset_asset_register, + layout, missing_line, notes, rule_cited, rule_label, rule_mark, template_data, + typeset_asset_register, }; use ledger_import::report::ReportMeta; use ledger_import::{PdfError, asset_register_pdf}; @@ -277,6 +278,50 @@ fn rule_words() { ); } +/// INVARIANT 9: the notes number the rules 1, 2, 3 (the general note +/// first and unnumbered, the built-in gain note 4), each rule's note +/// cites that rule, and the PDF marks each recorded line's rule with its +/// note's number; a line with no rule recorded has no mark. +#[test] +fn rule_marks_match_their_notes() { + use TaxBasisRule::*; + let rules = [Cost, Carryover, PersonalUseConversion]; + assert_eq!(rules.map(rule_mark), ["1", "2", "3"]); + let n = notes(); + let marks: Vec<&str> = n.iter().map(|n| n.mark.as_str()).collect(); + assert_eq!(marks, ["", "1", "2", "3", "4"]); + for rule in rules { + let note = n.iter().find(|n| n.mark == rule_mark(rule)).unwrap(); + let cites = format!("{}: ", rule_cited(rule)); + assert!(note.text.starts_with(&cites), "{rule:?}: {:?}", note.text); + } + // The template's input: each line its rule's mark, in register order. + let r = sample(); + let data = template_data(&r, &meta()); + let marks: Vec<&str> = data["lines"] + .as_array() + .unwrap() + .iter() + .map(|l| l["rule_mark"].as_str().unwrap()) + .collect(); + assert_eq!(marks, ["1", "2", "1", "3", ""]); + // The PDF: the mark right after the rule, the note under its number. + let t = all_text(&r, &meta()); + for s in [ + "(Ryzen 7 7840U, 64 GB, 4 TB) Cost 1 2,500.00", + "MacBook Air Carryover 2 1,299.00", + "Old monitor Cost 1 400.00", + "Sony A7 IV camera Personal-use conversion 3 1,650.00", + "Standing desk Not recorded 800.00", + "1 Cost: property the company bought", + "2 Carryover (IRC §723): property a member contributes", + "3 Personal-use conversion (Treas. Reg. §1.168(i)-4(b)): property converted", + "4 Built-in gain (loss):", + ] { + assert!(t.contains(s), "{s:?} missing from {t:?}"); + } +} + /// INVARIANT 4: an empty register renders a "No fixed assets" line, its /// totals, and the reconciliation. #[test] diff --git a/crates/ledger-import/tests/spec_import.rs b/crates/ledger-import/tests/spec_import.rs index 97576d0..4b435d0 100644 --- a/crates/ledger-import/tests/spec_import.rs +++ b/crates/ledger-import/tests/spec_import.rs @@ -2828,6 +2828,63 @@ fn xlsx_statement_layout_and_rules() { assert!(w.contains_key(&1) && w.contains_key(&2), "{w:?}"); } +/// INVARIANT 5's blank cells: a missing amount on a ruled row (a subtotal +/// under amounts, a total) is a blank cell in that row's ruled amount +/// format; on any other row (a line, a subtotal with nothing above it) +/// it writes no cell at all. +#[test] +fn xlsx_missing_amounts_are_cells_only_on_ruled_rows() { + let row = |kind, label: &str, amounts: Vec>| Row { + kind, + label: label.into(), + amounts, + currency: false, + rule_above: false, + }; + let st = report::statement( + "Blanks", + "July 2025".into(), + &["Book", "Tax"], + vec![ + row(RowKind::Section, "Income", vec![]), + row(RowKind::Line, "Sales", vec![Some(cents(100)), None]), + row(RowKind::Line, "Fees", vec![Some(cents(-5)), None]), + row( + RowKind::Subtotal, + "Total income", + vec![Some(cents(95)), None], + ), + row(RowKind::Spacer, "", vec![]), + row(RowKind::Section, "Empty", vec![]), + row(RowKind::Subtotal, "Total empty", vec![Some(cents(0)), None]), + row(RowKind::Spacer, "", vec![]), + row(RowKind::Total, "Net", vec![Some(cents(95)), None]), + ], + ); + let ruled: Vec<&str> = st + .rows + .iter() + .filter(|r| r.rule_above) + .map(|r| r.label.as_str()) + .collect(); + assert_eq!(ruled, ["Total income", "Net"]); + let bytes = report_to_xlsx(&XlsxReport::Statement(&st), &meta()).unwrap(); + let cells = sheet_cells(&bytes); + let styles = cell_styles(&bytes); + for r in st.rows.iter().filter(|r| !r.amounts.is_empty()) { + let at = row_of(&cells, &r.label, 0).unwrap(); + assert!(cells.contains_key(&(at, 1)), "{:?}: its amount", r.label); + assert!(!cells.contains_key(&(at, 2)), "{:?}: no value", r.label); + if r.rule_above { + let blank = styles.get(&(at, 2)); + assert!(blank.is_some_and(|s| s.top_border), "{:?}", r.label); + assert_eq!(blank, styles.get(&(at, 1)), "{:?}: as its amount", r.label); + } else { + assert!(!styles.contains_key(&(at, 2)), "{:?}: no cell", r.label); + } + } +} + /// INVARIANT 4 on the general ledger and journal: amount headings /// right-aligned and the others not, accounts one empty row apart, and the /// footer one row below (also with nothing to show). @@ -3217,12 +3274,57 @@ fn asset_register_xlsx_cells() { assert!( text_cell(&c, (22, 0)).starts_with("Book value is the fair market value at contribution") ); + for (row, start) in [ + (23, "1. Cost: "), + (24, "2. Carryover (IRC §723): "), + ( + 25, + "3. Personal-use conversion (Treas. Reg. §1.168(i)-4(b)): ", + ), + (26, "4. Built-in gain (loss): "), + ] { + assert!(text_cell(&c, (row, 0)).starts_with(start), "note {start:?}"); + } let footer = c .values() .any(|v| matches!(v, Cell::Text(t) if t.contains("Journal seq 42"))); assert!(footer, "footer present"); } +/// asset_register INVARIANTS 2 and 5 as laid out (crate::xlsx INVARIANTS +/// 4-5): the headings over the amounts and over the life in months +/// right-aligned and the others not, and the totals row ruled across +/// every column after its label in the totals format, blank cells and +/// totals alike; the asset rows above it aren't ruled. +#[test] +fn asset_register_xlsx_headings_and_totals_rule() { + let x = ledger_import::asset_register_xlsx(&asset_register_fixture(), &meta()).unwrap(); + let c = sheet_cells(&x); + let styles = cell_styles(&x); + let right_aligned = [4, 5, 6, 7, 8, 10, 11, 12]; + for col in 0..14 { + let right = styles[&(5, col)].halign.as_deref() == Some("right"); + assert_eq!(right, right_aligned.contains(&col), "heading column {col}"); + } + assert_eq!(text_cell(&c, (9, 0)), "Total, assets held"); + assert!(!styles[&(9, 0)].top_border, "the label isn't ruled"); + let total = &styles[&(9, 4)]; + assert!(total.top_border, "{total:?}"); + for col in 1..14 { + assert_eq!( + styles.get(&(9, col)), + Some(total), + "totals row column {col}" + ); + } + for row in 6..=8 { + for col in 0..14 { + let ruled = styles.get(&(row, col)).is_some_and(|s| s.top_border); + assert!(!ruled, "asset row {row} column {col}"); + } + } +} + /// asset_register INVARIANT 4: an empty register says so, still totals, /// and still reconciles (the books may carry unregistered cost). #[test] diff --git a/docs/testing.md b/docs/testing.md index 8003400..f9b3964 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -30,14 +30,14 @@ promise. | Benchmarks | a baseline (6) | `ledger-core/benches/books.rs`, `ledger-store/benches/store.rs` | `just bench` | ~2 min | none (no budget, by decision) | | Mutation | can the tests fail? | `ledger-core` (2026-10-13: 1,011 caught + 13 timeouts of 1,024 viable, 0 missed) | `just mutants` | ~3 h | weekly | | Mutation (store) | can the tests fail? | `ledger-store` (2026-10-13: 357 caught + 1 timeout of 365 viable, from 82%; three of its seven survivors then killed; four equivalent left: `sort_value`'s two arms, `OpenFlags` `|` vs `^`, the long form of a sweepless close) | `just mutants-store` | ~12 min | weekly | -| Mutation (import) | can the tests fail? | `ledger-import` (2026-10-13: from 91 of 171 (53%) to 208 of 239 viable on the full run; the 31 survivors (XLSX layout, borders, alignment, exact-text grouping, IIF transliteration, tax-summary optional sections, day-equals-month dates) then killed by targeted tests, re-run clean but one: a blank amount cell written unbordered on an unruled row, invisible, so equivalent) | `just mutants-import` | ~1 h | weekly | +| Mutation (import) | can the tests fail? | `ledger-import` (2026-10-13: from 91 of 171 (53%) to 208 of 239 viable on the full run; the 31 survivors (XLSX layout, borders, alignment, exact-text grouping, IIF transliteration, tax-summary optional sections, day-equals-month dates) then killed by targeted tests, re-run clean but one: a blank amount cell written unbordered on an unruled row, invisible, so equivalent; 2026-10-19, the first scheduled run in CI: 375 tested, 327 caught, 42 unviable, 6 missed (the tax-basis rule marks, a heading's alignment, the totals row's blank cells, and that blank cell, after all: a ruled row writes it, an unruled one writes none), all six then caught by `rule_marks_match_their_notes`, `asset_register_xlsx_headings_and_totals_rule` and `xlsx_missing_amounts_are_cells_only_on_ruled_rows`, each checked against its mutant by hand) | `just mutants-import` | ~2 h | weekly | | Mutation (roles, reload) | can the tests fail? | control roles and settings reload (2026-10-15, targeted: the role paths in `ledger-core` 30 caught of 30 viable; `ledger-store`'s journaling and replay of `role.assigned` 20 of 20; `ledger-mcp` settings, reload, roles, digest 28 of 28) | `cargo mutants -F` on those functions (see the commit) | ~15 min | with the crate's weekly run | | Mutation (config) | can the tests fail? | `ledger-config` (2026-10-14: 55 caught of 55 viable) | `just mutants-config` | minutes | weekly | | Mutation (MCP) | can the tests fail? | `ledger-mcp` (2026-10-14: from 197 caught + 5 timeouts of 328 viable (62%) to 314 caught + 2 timeouts of 331; of its 15 survivors, five then killed by targeted tests (the settings file beside the books, year earnings with an open period and with last year's partial close, year 1, a failed flush), one dead guard removed (`profit_and_loss_by_class` skipping placeholders, which never take postings), nine equivalent: `money` rescaling an amount already at two places, the lock wait's deadline instant, `year_earnings` starting the day the year starts either way, the bank import's rehearsal sides (a line's side can't make a draft refuse) and its zero-amount side (zero rows are dropped by the parser), a disposal's zero gain adding zero, and the CPA package index summing debits instead of credits (entries balance)) | `just mutants-mcp` | ~1 h | weekly | | Mutation (GUI) | can the tests fail? | `ledger-gui` view model and formatting (2026-10-14: 140 caught of 176 viable on the full run, then 92 of 99 on a targeted rerun of the money paths after new tests; of its 7 survivors, one then killed by a test (burn over exactly one month, confirmed by a rerun of `compute_kpis`'s 48 mutants), two dead arms removed (`parties`' bill arms: the ledger tags bill entries with their vendor, which wins), four equivalent: an entry's debits vs its credits (they balance), the three-month burn boundary (both paths give 3), burn over zero covered months (only when the first entry is after the window, which then holds no loss), and `compact`'s sign test at zero (zero prints "$0" unsigned either way)) | retired 2026-10-18 (owner: it crushed the machine; for the viewer, test its two musts directly: it never writes the books, `tests/read_only.rs`, and every figure it shows is the engine's, `tests/view_model.rs`) | | never | CI: `.github/workflows/check.yml` runs `just check` on every push; -`scheduled.yml` runs proofs, the oracle, fuzzing, and mutation weekly. +`scheduled.yml` runs proofs, the oracle, fuzzing, and mutation weekly. Mutation is split into shards, one job each (`just mutants-mcp 3/16` runs the fourth of sixteen), because GitHub stops a job at six hours and a whole crate takes longer on its runners (2026-10-19, the first scheduled run: the core and MCP runs were stopped at the limit). Test timeouts are three times each run's own baseline, never under the recipe's floor, so a slower runner doesn't report false timeouts. The viewer has no mutation job. Seeds and failure databases: proptest's `*.proptest-regressions` files are checked in and replay on every run. The GnuCash oracle's books come diff --git a/justfile b/justfile index 15a82ee..4328f00 100644 --- a/justfile +++ b/justfile @@ -88,40 +88,45 @@ miri: # Where cargo-mutants copies and builds the tree for each mutant (several # GB per job): the user's cache on disk, never /tmp, which is often RAM. # A mutant only needs some test to fail, so proptest doesn't shrink. +# +# Every mutation recipe takes a shard, "k/n" (default: all of it), so CI +# can spread a crate over parallel jobs. Test timeouts are three times the +# run's own baseline, never under the floor given: a slower machine gets +# longer ones instead of false timeouts. mutants_tmp := env_var_or_default("XDG_CACHE_HOME", env_var("HOME") + "/.cache") + "/wanderbooks/mutants" # Mutation testing of the core: do the tests catch planted bugs? -mutants: sweep-test-tmp +mutants shard="0/1": sweep-test-tmp # Core and store tests (the store's replay tests reach Ledger::replay_*); # the whole workspace would rebuild the viewer for every mutant. mkdir -p {{mutants_tmp}} - PROPTEST_MAX_SHRINK_ITERS=0 TMPDIR={{mutants_tmp}} cargo mutants -p ledger-core --exclude crates/ledger-core/src/proofs.rs --test-package ledger-core --test-package ledger-store -j 4 --timeout 120 + PROPTEST_MAX_SHRINK_ITERS=0 TMPDIR={{mutants_tmp}} cargo mutants -p ledger-core --exclude crates/ledger-core/src/proofs.rs --test-package ledger-core --test-package ledger-store -j 4 --timeout-multiplier 3 --minimum-test-timeout 120 --shard {{shard}} @just sweep-test-tmp # Mutation testing of the store. Fewer random cases per mutant (each needs # a run that catches it, not the deep sweep). -mutants-store: sweep-test-tmp +mutants-store shard="0/1": sweep-test-tmp mkdir -p {{mutants_tmp}} - STORE_CASES=64 PROPTEST_MAX_SHRINK_ITERS=0 TMPDIR={{mutants_tmp}} cargo mutants -p ledger-store --test-package ledger-store -j 3 --timeout 300 -o mutants-store + STORE_CASES=64 PROPTEST_MAX_SHRINK_ITERS=0 TMPDIR={{mutants_tmp}} cargo mutants -p ledger-store --test-package ledger-store -j 3 --timeout-multiplier 3 --minimum-test-timeout 300 --shard {{shard}} -o mutants-store @just sweep-test-tmp # Mutation testing of the importers and report renderers. -mutants-import: sweep-test-tmp +mutants-import shard="0/1": sweep-test-tmp mkdir -p {{mutants_tmp}} - PROPTEST_MAX_SHRINK_ITERS=0 TMPDIR={{mutants_tmp}} cargo mutants -p ledger-import --test-package ledger-import -j 4 --timeout 300 -o mutants-import + PROPTEST_MAX_SHRINK_ITERS=0 TMPDIR={{mutants_tmp}} cargo mutants -p ledger-import --test-package ledger-import -j 4 --timeout-multiplier 3 --minimum-test-timeout 300 --shard {{shard}} -o mutants-import @just sweep-test-tmp # Mutation testing of the settings parser and the receipt policy. -mutants-config: sweep-test-tmp +mutants-config shard="0/1": sweep-test-tmp mkdir -p {{mutants_tmp}} - PROPTEST_MAX_SHRINK_ITERS=0 TMPDIR={{mutants_tmp}} cargo mutants -p ledger-config --test-package ledger-config -j 4 --timeout 120 -o mutants-config + PROPTEST_MAX_SHRINK_ITERS=0 TMPDIR={{mutants_tmp}} cargo mutants -p ledger-config --test-package ledger-config -j 4 --timeout-multiplier 3 --minimum-test-timeout 120 --shard {{shard}} -o mutants-config @just sweep-test-tmp # Mutation testing of the MCP server (every mutant runs the stdio suites; # fewer random cases each, as for the store). -mutants-mcp: sweep-test-tmp +mutants-mcp shard="0/1": sweep-test-tmp mkdir -p {{mutants_tmp}} - MCP_CASES=16 PROPTEST_MAX_SHRINK_ITERS=0 TMPDIR={{mutants_tmp}} cargo mutants -p ledger-mcp --test-package ledger-mcp -j 4 --timeout 600 -o mutants-mcp + MCP_CASES=16 PROPTEST_MAX_SHRINK_ITERS=0 TMPDIR={{mutants_tmp}} cargo mutants -p ledger-mcp --test-package ledger-mcp -j 4 --timeout-multiplier 3 --minimum-test-timeout 600 --shard {{shard}} -o mutants-mcp @just sweep-test-tmp # No mutation runs on the viewer (owner, 2026-10-18: they crushed the