From c6527280e708fcc3f11259f9fce96f473614ef64 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Mon, 26 Oct 2026 16:07:53 -0600 Subject: [PATCH] fix(parties): read every separator a TIN can be written with (P7) A taxpayer identification number can be written 123,45,6789 or (123) 45-6789 as easily as 123-45-6789, and the scanner only knew the second: a comma-grouped number slipped through every party field, every refusal that quotes its input and the access log's masking, which is how a full TIN could reach a result the books promise never shows one (rules.md B-26, I-7). The scanner now reads a comma or a parenthesis between a TIN's 3-2-4 or 2-7 groups, and any run of spaces counts once however long it is, so 12 3456789 is an EIN. Commas and parentheses join only those groups: an amount's thousands separators (1,234,567.89 is 1-3-3-2) still read as an amount, and a nine-digit D-U-N-S number stays its own setting's kind (company.duns). The old run reading is unchanged for everything it found, which the property asserts; replay still reads a journaled line as P2 did (B-28). The truth table and the ASCII-equality property pinned the old separator set, so they were updated with it: the property is now has_full_tin_never_forgets_a_tin (what was found before is still found). No journal, entry or figure changes: this is a text scanner only. --- crates/ledger-core/src/parties.rs | 133 ++++++++++++++++++----- crates/ledger-core/tests/properties.rs | 57 ++++++++++ crates/ledger-core/tests/spec_parties.rs | 105 ++++++++++-------- 3 files changed, 224 insertions(+), 71 deletions(-) diff --git a/crates/ledger-core/src/parties.rs b/crates/ledger-core/src/parties.rs index 514c8d9..ad8678e 100644 --- a/crates/ledger-core/src/parties.rs +++ b/crates/ledger-core/src/parties.rs @@ -15,8 +15,10 @@ //! 1. No party field holds a full taxpayer identification number: any //! run of nine digits, in any script, separated or not (`has_full_tin`: //! up to three spaces, hyphens, dots, slashes or underscores, in any -//! of their Unicode forms, between neighbors) is refused. A W-9's TIN -//! is kept as its last four digits only. +//! of their Unicode forms, between neighbors, any run of spaces +//! counting once; or a 3-2-4 or 2-7 group with commas or parentheses +//! between its groups) is refused. A W-9's TIN is kept as its last +//! four digits only. //! 2. W-9 facts need a 1099 role (contractor or landlord); W-4 facts need //! the employee role. Taxpayer-ID facts (`TinFacts`: the TIN's type, //! last four, and vault record) are allowed for any role: members @@ -408,14 +410,65 @@ fn tin_invisible(c: char) -> bool { } /// The most separators that may sit between two of a TIN's digits -/// (`" - "` is three); invisible characters ([`tin_invisible`]) don't -/// count. +/// (`" - "` is three); a run of spaces counts once however long it is, +/// and invisible characters ([`tin_invisible`]) don't count. const MAX_TIN_GAP: usize = 3; +/// Whether `c` is one of the spaces between a TIN's digits: a run of +/// them, however long, counts as one separator ([`tin_gap`]), so +/// `12 3456789` reads as an EIN. +fn tin_space(c: char) -> bool { + c.is_whitespace() && tin_separator(c) +} + +/// Whether `c` is a comma or a parenthesis: these separate a TIN's +/// *groups* (`123,45,6789`, `(123) 45-6789`) and never an ungrouped run, +/// so an amount's thousands separators (`1,234,567.89`, 1-3-3-2) are not +/// a TIN ([`grouped_tin_runs`]). +fn grouped_tin_separator(c: char) -> bool { + matches!(c, ',' | '(' | ')') +} + +/// The gap from `chars[at..]` between two digits: how many characters it +/// takes, or None when the next thing that isn't a separator or invisible +/// isn't a digit, or it holds more than [`MAX_TIN_GAP`] separators. A run +/// of spaces ([`tin_space`]) counts once, however long; invisible +/// characters count not at all; `grouped` also reads a comma, a +/// parenthesis ([`grouped_tin_separator`]). +fn tin_gap(chars: &[char], at: usize, grouped: bool) -> Option { + let (mut i, mut count, mut in_spaces) = (at, 0, false); + while i < chars.len() { + let c = chars[i]; + if tin_invisible(c) { + i += 1; + continue; + } + if tin_space(c) { + if !in_spaces { + count += 1; + in_spaces = true; + } + i += 1; + continue; + } + if tin_separator(c) || (grouped && grouped_tin_separator(c)) { + count += 1; + in_spaces = false; + i += 1; + continue; + } + break; + } + (i > at && count <= MAX_TIN_GAP && chars.get(i).is_some_and(|c| c.is_numeric())) + .then_some(i - at) +} + /// Each run in `chars` that holds nine digits or more, as (first, last) /// char indices of its digits: digits with at most [`MAX_TIN_GAP`] -/// separators ([`tin_separator`]) between neighbors, and any number of -/// invisible characters. +/// separators ([`tin_separator`]) between neighbors, any run of spaces +/// counting once, and any number of invisible characters. Also every +/// 3-2-4 or 2-7 group ([`grouped_tin_runs`]), whose groups a comma or a +/// parenthesis may join. fn full_tin_runs(chars: &[char]) -> Vec<(usize, usize)> { let mut runs = Vec::new(); let mut i = 0; @@ -432,21 +485,9 @@ fn full_tin_runs(chars: &[char]) -> Vec<(usize, usize)> { j += 1; continue; } - let between = chars[j..] - .iter() - .take_while(|c| tin_separator(**c) || tin_invisible(**c)) - .count(); - let gap = chars[j..j + between] - .iter() - .filter(|c| !tin_invisible(**c)) - .count(); - if between > 0 - && gap <= MAX_TIN_GAP - && chars.get(j + between).is_some_and(|c| c.is_numeric()) - { - j += between; - } else { - break; + match tin_gap(chars, j, false) { + Some(consumed) => j += consumed, + None => break, } } if digits >= 9 { @@ -454,17 +495,57 @@ fn full_tin_runs(chars: &[char]) -> Vec<(usize, usize)> { } i = last + 1; } + runs.extend(grouped_tin_runs(chars)); + runs.sort_unstable(); + runs +} + +/// Each 3-2-4 or 2-7 group in `chars`, as (first, last) char indices of +/// its digits: a TIN's shapes (`123-45-6789`, `12-3456789`, +/// `123,45,6789`, `(123) 45-6789`). A comma or a parenthesis separates +/// these groups and no other run, so the amount `1,234,567.89` (1-3-3-2) +/// is no TIN however it is grouped. +fn grouped_tin_runs(chars: &[char]) -> Vec<(usize, usize)> { + const SHAPES: [&[usize]; 2] = [&[3, 2, 4], &[2, 7]]; + let mut runs = Vec::new(); + for at in 0..chars.len() { + if !chars[at].is_numeric() { + continue; + } + 'shapes: for shape in SHAPES { + let mut i = at; + for (n, &len) in shape.iter().enumerate() { + if n > 0 { + match tin_gap(chars, i, true) { + Some(consumed) => i += consumed, + None => continue 'shapes, + } + } + for _ in 0..len { + if !chars.get(i).is_some_and(|c| c.is_numeric()) { + continue 'shapes; + } + i += 1; + } + } + runs.push((at, i - 1)); + } + } runs } /// Whether `text` holds a run of nine digits, with at most three /// separators (a space, hyphen, dot, slash or underscore, in any of -/// their Unicode forms) between neighbors, invisible -/// characters not counted: an SSN, EIN, or ITIN however it's written -/// (`291-12-2846`, `291.12.2846`, `291 - 12 - 2846`, `２９１－１２－２８４６`). +/// their Unicode forms, and any run of spaces counting once) between +/// neighbors, invisible characters not counted, or one of a TIN's groups +/// (3-2-4 or 2-7) with a comma or a parenthesis between them: an SSN, +/// EIN, or ITIN however it's written (`291-12-2846`, `291.12.2846`, +/// `291 - 12 - 2846`, `291,12,2846`, `(291) 12-2846`, `２９１－１２－２８４６`). /// Any Unicode numeral counts as a digit (full-width, Arabic-Indic, ...), -/// so a number can't slip through in another script. In ASCII, exactly -/// the space, hyphen, dot, slash and underscore separate. +/// so a number can't slip through in another script. In ASCII, the space, +/// hyphen, dot, slash and underscore separate, and so do commas and +/// parentheses between a TIN's groups: a comma-grouped amount +/// (`1,234,567.89`) is no TIN. pub fn has_full_tin(text: &str) -> bool { let chars: Vec = text.chars().collect(); if REPLAYING.with(std::cell::Cell::get) { diff --git a/crates/ledger-core/tests/properties.rs b/crates/ledger-core/tests/properties.rs index e0fcf82..7257170 100644 --- a/crates/ledger-core/tests/properties.rs +++ b/crates/ledger-core/tests/properties.rs @@ -1141,3 +1141,60 @@ proptest! { } } } + +proptest! { + #![proptest_config(ProptestConfig::with_cases(2000))] + + /// rules.md B-26 (P7): a TIN is found and masked however its groups + /// are separated — a hyphen, a dot, a comma, a parenthesis, a slash, + /// an underscore, or any run of spaces — and what merely looks like + /// one is not: an amount's thousands separators (1,234,567.89 is + /// 1-3-3-2, no TIN's shape), a longer number holding a TIN's digits, + /// and the company's own nine-digit D-U-N-S number, which is its own + /// setting's kind (company.duns), never a party field. The ACH payment + /// methods' lines get the same exemption in P4b S4's Kind; until then + /// the filing scan's narrow shapes keep a bank export's routing + /// numbers out of the rule (rules.md B-26). + #[test] + fn tin_shapes_with_any_separator_are_found( + sep in prop::sample::select(vec![ + "-", " ", " ", " ", ".", ",", "/", "_", " - ", ", ", ") ", "(", ")", ",\u{a0}", + ]), + which in 0usize..2, + ) { + let shapes: [&[&str]; 2] = [&["187", "05", "1907"], &["12", "3456789"]]; + let tin = shapes[which].join(sep); + prop_assert!(ledger_core::parties::has_full_tin(&tin), "{:?}", tin); + let masked = ledger_core::parties::redact_full_tins(&tin); + prop_assert_eq!( + masked.chars().filter(|c| *c == '•').count(), + 9, + "{:?}: {:?}", + tin, + masked + ); + prop_assert!( + !ledger_core::parties::has_full_tin(&masked), + "{:?}: {:?}", + tin, + masked + ); + + // No false positive on an amount. + prop_assert!(!ledger_core::parties::has_full_tin("1,234,567.89")); + let amount = ledger_core::parties::redact_full_tins("1,234,567.89"); + prop_assert_eq!(amount.as_ref(), "1,234,567.89"); + // Nor on the company's D-U-N-S number: the key's own kind takes a + // nine-digit number, bare or printed 2-3-4. + let duns = ledger_core::settings::KEYS + .iter() + .find(|k| k.name == "company.duns") + .expect("the books know company.duns"); + prop_assert!( + ledger_core::settings::check(duns, &serde_json::json!("149976972")).is_ok() + ); + prop_assert!( + ledger_core::settings::check(duns, &serde_json::json!("14-997-6972")).is_ok() + ); + } +} diff --git a/crates/ledger-core/tests/spec_parties.rs b/crates/ledger-core/tests/spec_parties.rs index f2fad5b..b75c26e 100644 --- a/crates/ledger-core/tests/spec_parties.rs +++ b/crates/ledger-core/tests/spec_parties.rs @@ -155,7 +155,9 @@ fn pay_line(name: &str, employee: Option) -> PayrollLine { /// Oracle: `has_full_tin` doc ("a run of nine digits, with at most three /// separators (space, hyphen, dot, slash, underscore, a Unicode dash or -/// no-break space) between neighbors") and INVARIANT 1. +/// no-break space) between neighbors, a run of spaces counting once; or a +/// TIN's 3-2-4 or 2-7 groups with commas or parentheses between them") and +/// INVARIANT 1. #[test] fn has_full_tin_truth_table() { let cases: &[(&str, bool)] = &[ @@ -177,15 +179,21 @@ fn has_full_tin_truth_table() { ("Bob 291 - 12 - 2846", true), // three separators ("291\u{2013}12\u{2013}2846", true), // en dashes ("291\u{a0}12\u{a0}2846", true), // no-break spaces - ("12 3456789", false), // four separators break the run: 2 + 7 - ("291, 12, 2846", false), // a comma is not a separator - ("12\t3456789", false), // a tab is not a space - ("12345678", false), // eight digits + ("12 3456789", true), // a run of spaces counts once + ("291, 12, 2846", true), // commas between a TIN's groups + ("291,12,2846", true), + ("12,3456789", true), // an EIN's 2-7 groups + ("(291) 12-2846", true), // parentheses around a group + ("12\t3456789", false), // a tab is not a space + ("12345678", false), // eight digits ("123-45-678", false), ("1234", false), ("", false), ("Acme LLC", false), ("Suite 100, 84101", false), // 3 + 5 digits, separated by ", " + ("1,234,567.89", false), // an amount: comma groups of three + ("123,456,789", false), // a comma-grouped number, not a TIN + ("(123) 456-7890", false), // a phone number: 3-3-4 ("-123456789-", true), ]; for (text, want) in cases { @@ -242,17 +250,20 @@ fn has_full_tin_unicode_digits() { assert!(has_full_tin("١٢٣٤٥٦٧٨٩")); // Arabic-Indic digits } -/// Oracle: `has_full_tin` doc (P3 review round 2): past ASCII, every form -/// of the five separators Unicode has separates (every Zs space, every +/// Oracle: `has_full_tin` doc (P3 review round 2; P7): past ASCII, every +/// form of the five separators Unicode has separates (every Zs space, every /// Dash character, the compatibility forms of the dot, slash and /// underscore; `docs/spec/sources/unicode-tin-text.md`, Unicode 17.0), /// and invisible characters (Default_Ignorable_Code_Point) don't count -/// toward the gap. A TIN typed in fullwidth forms, Arabic-Indic digits, -/// with an en dash, a non-breaking hyphen, a Hebrew maqaf, any space, or -/// with right-to-left marks around its hyphens is still a full TIN; line -/// breaks, commas and a fourth visible separator still end a run. -/// (Before, "１２３－４５－６７８９" passed: the fullwidth hyphen-minus wasn't -/// a separator, nor the ideographic or thin space.) +/// toward the gap. A run of spaces counts once, however long, and a comma +/// or a parenthesis separates a TIN's 3-2-4 or 2-7 groups (ASCII only). +/// A TIN typed in fullwidth forms, Arabic-Indic digits, with an en dash, a +/// non-breaking hyphen, a Hebrew maqaf, any space, a comma or a +/// parenthesis, or with right-to-left marks around its hyphens is still a +/// full TIN; line breaks, commas in an ungrouped run, tabs and a fourth +/// other visible separator still end a run. (Before P3, "１２３－４５－６７８９" +/// passed: the fullwidth hyphen-minus wasn't a separator, nor the +/// ideographic or thin space.) #[test] fn has_full_tin_reads_every_script_and_separator() { let tins = [ @@ -264,21 +275,26 @@ fn has_full_tin_reads_every_script_and_separator() { "\u{661}\u{662}\u{663}-\u{664}\u{665}-\u{666}\u{667}\u{668}\u{669}", "\u{6F1}\u{6F2}\u{6F3}\u{6F4}\u{6F5}\u{6F6}\u{6F7}\u{6F8}\u{6F9}", "\u{967}\u{968}\u{969}\u{2013}\u{96A}\u{96B}\u{2013}\u{96C}\u{96D}\u{96E}\u{96F}", - "291\u{2013}12\u{2013}2846", // en dash - "291\u{2011}12\u{2011}2846", // non-breaking hyphen - "291\u{2012}12\u{2012}2846", // figure dash - "291\u{2015}12\u{2015}2846", // horizontal bar - "291\u{5BE}12\u{5BE}2846", // Hebrew maqaf - "291\u{301C}12\u{301C}2846", // wave dash - "291\u{A0}12\u{A0}2846", // no-break space - "291\u{3000}12\u{3000}2846", // ideographic space - "291\u{2009}12\u{2009}2846", // thin space - "291\u{202F}12\u{202F}2846", // narrow no-break space - "291 \u{2014} 12 \u{2014} 2846", // three: space, em dash, space - "291\u{FF0E}12\u{FF0E}2846", // fullwidth full stop - "291\u{2024}12\u{2024}2846", // one dot leader - "291\u{FF0F}12\u{FF0F}2846", // fullwidth solidus - "291\u{FF3F}12\u{FF3F}2846", // fullwidth low line + "291\u{2013}12\u{2013}2846", // en dash + "291\u{2011}12\u{2011}2846", // non-breaking hyphen + "291\u{2012}12\u{2012}2846", // figure dash + "291\u{2015}12\u{2015}2846", // horizontal bar + "291\u{5BE}12\u{5BE}2846", // Hebrew maqaf + "291\u{301C}12\u{301C}2846", // wave dash + "291\u{A0}12\u{A0}2846", // no-break space + "291\u{3000}12\u{3000}2846", // ideographic space + "291\u{2009}12\u{2009}2846", // thin space + "291\u{202F}12\u{202F}2846", // narrow no-break space + "291 \u{2014} 12 \u{2014} 2846", // three: space, em dash, space + "12\u{3000}\u{3000}\u{3000}\u{3000}3456789", // a run of spaces counts once + "291\u{FF0E}12\u{FF0E}2846", // fullwidth full stop + "291\u{2024}12\u{2024}2846", // one dot leader + "291\u{FF0F}12\u{FF0F}2846", // fullwidth solidus + "291\u{FF3F}12\u{FF3F}2846", // fullwidth low line + "291,12,2846", // commas between the groups + "291, 12, 2846", // a comma and a space + "(291) 12-2846", // parentheses around a group + "12,3456789", // an EIN's groups // Zero-width spaces between every digit. "2\u{200B}9\u{200B}1\u{200B}1\u{200B}2\u{200B}2\u{200B}8\u{200B}4\u{200B}6", "291\u{200F}-\u{200F}12\u{200F}-\u{200F}2846", // right-to-left marks around hyphens @@ -308,24 +324,25 @@ fn has_full_tin_reads_every_script_and_separator() { "•••\u{200F}-\u{200F}••-•••• ok" ); let not = [ - "12\u{3000}\u{3000}\u{3000}\u{3000}3456789", // four visible separators - "12 \u{2014} \u{200B}-3456789", // four, an invisible one among them - "291\u{2028}12\u{2028}2846", // line separators end a run - "291\u{2029}12\u{2029}2846", // paragraph separators - "291\u{85}12\u{85}2846", // next line - "291\u{FF0C}12\u{FF0C}2846", // fullwidth commas - "291\u{3001}12\u{3001}2846", // ideographic commas + "12 \u{2014} \u{200B}-3456789", // four visible, an invisible one among them + "291\u{2028}12\u{2028}2846", // line separators end a run + "291\u{2029}12\u{2029}2846", // paragraph separators + "291\u{85}12\u{85}2846", // next line + "291\u{FF0C}12\u{FF0C}2846", // fullwidth commas are not a group's comma + "291\u{3001}12\u{3001}2846", // ideographic commas either + "1,234,567.89", // an amount: comma groups of three + "(123) 456-7890", // a phone number: 3-3-4 "\u{FF11}\u{FF12}\u{FF13}\u{FF14}\u{FF15}\u{FF16}\u{FF17}\u{FF18}", // eight digits - "291\u{200B}12284", // eight, invisibly joined + "291\u{200B}12284", // eight, invisibly joined ]; for t in not { assert!(!has_full_tin(t), "has_full_tin({t:?})"); } } -/// The reading before P3 review round 2, the oracle for ASCII: past it, -/// only the no-break space and a few dashes separated, and every -/// separator counted. +/// The reading before P3 review round 2, the oracle for what P7 must +/// never narrow: P7 only widens what reads as a TIN (commas and +/// parentheses between a TIN's groups, any run of spaces counting once). fn has_full_tin_before(text: &str) -> bool { let sep = |c: char| { matches!( @@ -366,15 +383,13 @@ fn has_full_tin_before(text: &str) -> bool { proptest! { #![proptest_config(ProptestConfig::with_cases(2000))] - /// `has_full_tin` reads ASCII exactly as it did (the fix only widens - /// what separates past ASCII), and what it found before in any text - /// it still finds: it only ever finds more. + /// What `has_full_tin` found before P7 it still finds: the fix only + /// widens the reading (commas and parentheses between a TIN's groups, + /// any run of spaces counting once), never narrows it. #[test] - fn has_full_tin_keeps_its_ascii_reading( - ascii in "[0-9 ./_\t\n\r,:a-]{0,40}", + fn has_full_tin_never_forgets_a_tin( any in "[0-9 ./_\t\n,a\u{A0}\u{2010}-\u{2015}\u{2212}\u{FF0D}\u{3000}\u{200B}\u{200F}\u{FF10}-\u{FF19}\u{660}-\u{669}\u{2028}-]{0,40}", ) { - prop_assert_eq!(has_full_tin(&ascii), has_full_tin_before(&ascii), "{:?}", ascii); if has_full_tin_before(&any) { prop_assert!(has_full_tin(&any), "{:?}", any); }