fix(parties): read every separator a TIN can be written with (P7)

A taxpayer identification number can be written 123,45,6789 or
(123) 45-6789 as easily as 123-45-6789, and the scanner only knew the
second: a comma-grouped number slipped through every party field, every
refusal that quotes its input and the access log's masking, which is how
a full TIN could reach a result the books promise never shows one
(rules.md B-26, I-7).

The scanner now reads a comma or a parenthesis between a TIN's 3-2-4 or
2-7 groups, and any run of spaces counts once however long it is, so
12    3456789 is an EIN. Commas and parentheses join only those groups:
an amount's thousands separators (1,234,567.89 is 1-3-3-2) still read as
an amount, and a nine-digit D-U-N-S number stays its own setting's kind
(company.duns). The old run reading is unchanged for everything it
found, which the property asserts; replay still reads a journaled line
as P2 did (B-28).

The truth table and the ASCII-equality property pinned the old separator
set, so they were updated with it: the property is now
has_full_tin_never_forgets_a_tin (what was found before is still found).
No journal, entry or figure changes: this is a text scanner only.

crates/ledger-core/src/parties.rs

@@ -15,8 +15,10 @@
 //!   1. No party field holds a full taxpayer identification number: any
 //!      run of nine digits, in any script, separated or not (`has_full_tin`:
 //!      up to three spaces, hyphens, dots, slashes or underscores, in any
-//!      of their Unicode forms, between neighbors) is refused. A W-9's TIN
+//!      of their Unicode forms, between neighbors, any run of spaces
-//!      is kept as its last four digits only.
+//!      counting once; or a 3-2-4 or 2-7 group with commas or parentheses
+//!      between its groups) is refused. A W-9's TIN is kept as its last
+//!      four digits only.
 //!   2. W-9 facts need a 1099 role (contractor or landlord); W-4 facts need
 //!      the employee role. Taxpayer-ID facts (`TinFacts`: the TIN's type,
 //!      last four, and vault record) are allowed for any role: members
@@ -408,14 +410,65 @@ fn tin_invisible(c: char) -> bool {
 }
 
 /// The most separators that may sit between two of a TIN's digits
-/// (`" - "` is three); invisible characters ([`tin_invisible`]) don't
+/// (`" - "` is three); a run of spaces counts once however long it is,
-/// count.
+/// and invisible characters ([`tin_invisible`]) don't count.
 const MAX_TIN_GAP: usize = 3;
 
+/// Whether `c` is one of the spaces between a TIN's digits: a run of
+/// them, however long, counts as one separator ([`tin_gap`]), so
+/// `12    3456789` reads as an EIN.
+fn tin_space(c: char) -> bool {
+    c.is_whitespace() && tin_separator(c)
+}
+
+/// Whether `c` is a comma or a parenthesis: these separate a TIN's
+/// *groups* (`123,45,6789`, `(123) 45-6789`) and never an ungrouped run,
+/// so an amount's thousands separators (`1,234,567.89`, 1-3-3-2) are not
+/// a TIN ([`grouped_tin_runs`]).
+fn grouped_tin_separator(c: char) -> bool {
+    matches!(c, ',' | '(' | ')')
+}
+
+/// The gap from `chars[at..]` between two digits: how many characters it
+/// takes, or None when the next thing that isn't a separator or invisible
+/// isn't a digit, or it holds more than [`MAX_TIN_GAP`] separators. A run
+/// of spaces ([`tin_space`]) counts once, however long; invisible
+/// characters count not at all; `grouped` also reads a comma, a
+/// parenthesis ([`grouped_tin_separator`]).
+fn tin_gap(chars: &[char], at: usize, grouped: bool) -> Option<usize> {
+    let (mut i, mut count, mut in_spaces) = (at, 0, false);
+    while i < chars.len() {
+        let c = chars[i];
+        if tin_invisible(c) {
+            i += 1;
+            continue;
+        }
+        if tin_space(c) {
+            if !in_spaces {
+                count += 1;
+                in_spaces = true;
+            }
+            i += 1;
+            continue;
+        }
+        if tin_separator(c) || (grouped && grouped_tin_separator(c)) {
+            count += 1;
+            in_spaces = false;
+            i += 1;
+            continue;
+        }
+        break;
+    }
+    (i > at && count <= MAX_TIN_GAP && chars.get(i).is_some_and(|c| c.is_numeric()))
+        .then_some(i - at)
+}
+
 /// Each run in `chars` that holds nine digits or more, as (first, last)
 /// char indices of its digits: digits with at most [`MAX_TIN_GAP`]
-/// separators ([`tin_separator`]) between neighbors, and any number of
+/// separators ([`tin_separator`]) between neighbors, any run of spaces
-/// invisible characters.
+/// counting once, and any number of invisible characters. Also every
+/// 3-2-4 or 2-7 group ([`grouped_tin_runs`]), whose groups a comma or a
+/// parenthesis may join.
 fn full_tin_runs(chars: &[char]) -> Vec<(usize, usize)> {
     let mut runs = Vec::new();
     let mut i = 0;
@@ -432,21 +485,9 @@ fn full_tin_runs(chars: &[char]) -> Vec<(usize, usize)> {
                 j += 1;
                 continue;
             }
-            let between = chars[j..]
+            match tin_gap(chars, j, false) {
-                .iter()
+                Some(consumed) => j += consumed,
-                .take_while(|c| tin_separator(**c) || tin_invisible(**c))
+                None => break,
-                .count();
-            let gap = chars[j..j + between]
-                .iter()
-                .filter(|c| !tin_invisible(**c))
-                .count();
-            if between > 0
-                && gap <= MAX_TIN_GAP
-                && chars.get(j + between).is_some_and(|c| c.is_numeric())
-            {
-                j += between;
-            } else {
-                break;
             }
         }
         if digits >= 9 {
@@ -454,17 +495,57 @@ fn full_tin_runs(chars: &[char]) -> Vec<(usize, usize)> {
         }
         i = last + 1;
     }
+    runs.extend(grouped_tin_runs(chars));
+    runs.sort_unstable();
+    runs
+}
+
+/// Each 3-2-4 or 2-7 group in `chars`, as (first, last) char indices of
+/// its digits: a TIN's shapes (`123-45-6789`, `12-3456789`,
+/// `123,45,6789`, `(123) 45-6789`). A comma or a parenthesis separates
+/// these groups and no other run, so the amount `1,234,567.89` (1-3-3-2)
+/// is no TIN however it is grouped.
+fn grouped_tin_runs(chars: &[char]) -> Vec<(usize, usize)> {
+    const SHAPES: [&[usize]; 2] = [&[3, 2, 4], &[2, 7]];
+    let mut runs = Vec::new();
+    for at in 0..chars.len() {
+        if !chars[at].is_numeric() {
+            continue;
+        }
+        'shapes: for shape in SHAPES {
+            let mut i = at;
+            for (n, &len) in shape.iter().enumerate() {
+                if n > 0 {
+                    match tin_gap(chars, i, true) {
+                        Some(consumed) => i += consumed,
+                        None => continue 'shapes,
+                    }
+                }
+                for _ in 0..len {
+                    if !chars.get(i).is_some_and(|c| c.is_numeric()) {
+                        continue 'shapes;
+                    }
+                    i += 1;
+                }
+            }
+            runs.push((at, i - 1));
+        }
+    }
     runs
 }
 
 /// Whether `text` holds a run of nine digits, with at most three
 /// separators (a space, hyphen, dot, slash or underscore, in any of
-/// their Unicode forms) between neighbors, invisible
+/// their Unicode forms, and any run of spaces counting once) between
-/// characters not counted: an SSN, EIN, or ITIN however it's written
+/// neighbors, invisible characters not counted, or one of a TIN's groups
-/// (`291-12-2846`, `291.12.2846`, `291 - 12 - 2846`, `２９１－１２－２８４６`).
+/// (3-2-4 or 2-7) with a comma or a parenthesis between them: an SSN,
+/// EIN, or ITIN however it's written (`291-12-2846`, `291.12.2846`,
+/// `291 - 12 - 2846`, `291,12,2846`, `(291) 12-2846`, `２９１－１２－２８４６`).
 /// Any Unicode numeral counts as a digit (full-width, Arabic-Indic, ...),
-/// so a number can't slip through in another script. In ASCII, exactly
+/// so a number can't slip through in another script. In ASCII, the space,
-/// the space, hyphen, dot, slash and underscore separate.
+/// hyphen, dot, slash and underscore separate, and so do commas and
+/// parentheses between a TIN's groups: a comma-grouped amount
+/// (`1,234,567.89`) is no TIN.
 pub fn has_full_tin(text: &str) -> bool {
     let chars: Vec<char> = text.chars().collect();
     if REPLAYING.with(std::cell::Cell::get) {

crates/ledger-core/tests/properties.rs

@@ -1141,3 +1141,60 @@ proptest! {
         }
     }
 }
+
+proptest! {
+    #![proptest_config(ProptestConfig::with_cases(2000))]
+
+    /// rules.md B-26 (P7): a TIN is found and masked however its groups
+    /// are separated — a hyphen, a dot, a comma, a parenthesis, a slash,
+    /// an underscore, or any run of spaces — and what merely looks like
+    /// one is not: an amount's thousands separators (1,234,567.89 is
+    /// 1-3-3-2, no TIN's shape), a longer number holding a TIN's digits,
+    /// and the company's own nine-digit D-U-N-S number, which is its own
+    /// setting's kind (company.duns), never a party field. The ACH payment
+    /// methods' lines get the same exemption in P4b S4's Kind; until then
+    /// the filing scan's narrow shapes keep a bank export's routing
+    /// numbers out of the rule (rules.md B-26).
+    #[test]
+    fn tin_shapes_with_any_separator_are_found(
+        sep in prop::sample::select(vec![
+            "-", " ", "  ", "     ", ".", ",", "/", "_", " - ", ", ", ") ", "(", ")", ",\u{a0}",
+        ]),
+        which in 0usize..2,
+    ) {
+        let shapes: [&[&str]; 2] = [&["187", "05", "1907"], &["12", "3456789"]];
+        let tin = shapes[which].join(sep);
+        prop_assert!(ledger_core::parties::has_full_tin(&tin), "{:?}", tin);
+        let masked = ledger_core::parties::redact_full_tins(&tin);
+        prop_assert_eq!(
+            masked.chars().filter(|c| *c == '•').count(),
+            9,
+            "{:?}: {:?}",
+            tin,
+            masked
+        );
+        prop_assert!(
+            !ledger_core::parties::has_full_tin(&masked),
+            "{:?}: {:?}",
+            tin,
+            masked
+        );
+
+        // No false positive on an amount.
+        prop_assert!(!ledger_core::parties::has_full_tin("1,234,567.89"));
+        let amount = ledger_core::parties::redact_full_tins("1,234,567.89");
+        prop_assert_eq!(amount.as_ref(), "1,234,567.89");
+        // Nor on the company's D-U-N-S number: the key's own kind takes a
+        // nine-digit number, bare or printed 2-3-4.
+        let duns = ledger_core::settings::KEYS
+            .iter()
+            .find(|k| k.name == "company.duns")
+            .expect("the books know company.duns");
+        prop_assert!(
+            ledger_core::settings::check(duns, &serde_json::json!("149976972")).is_ok()
+        );
+        prop_assert!(
+            ledger_core::settings::check(duns, &serde_json::json!("14-997-6972")).is_ok()
+        );
+    }
+}

crates/ledger-core/tests/spec_parties.rs

@@ -155,7 +155,9 @@ fn pay_line(name: &str, employee: Option<PartyId>) -> PayrollLine {
 
 /// Oracle: `has_full_tin` doc ("a run of nine digits, with at most three
 /// separators (space, hyphen, dot, slash, underscore, a Unicode dash or
-/// no-break space) between neighbors") and INVARIANT 1.
+/// no-break space) between neighbors, a run of spaces counting once; or a
+/// TIN's 3-2-4 or 2-7 groups with commas or parentheses between them") and
+/// INVARIANT 1.
 #[test]
 fn has_full_tin_truth_table() {
     let cases: &[(&str, bool)] = &[
@@ -177,15 +179,21 @@ fn has_full_tin_truth_table() {
         ("Bob 291 - 12 - 2846", true),       // three separators
         ("291\u{2013}12\u{2013}2846", true), // en dashes
         ("291\u{a0}12\u{a0}2846", true),     // no-break spaces
-        ("12    3456789", false),            // four separators break the run: 2 + 7
+        ("12    3456789", true),             // a run of spaces counts once
-        ("291, 12, 2846", false),            // a comma is not a separator
+        ("291, 12, 2846", true),             // commas between a TIN's groups
-        ("12\t3456789", false),              // a tab is not a space
+        ("291,12,2846", true),
-        ("12345678", false),                 // eight digits
+        ("12,3456789", true),    // an EIN's 2-7 groups
+        ("(291) 12-2846", true), // parentheses around a group
+        ("12\t3456789", false),  // a tab is not a space
+        ("12345678", false),     // eight digits
         ("123-45-678", false),
         ("1234", false),
         ("", false),
         ("Acme LLC", false),
         ("Suite 100, 84101", false), // 3 + 5 digits, separated by ", "
+        ("1,234,567.89", false),     // an amount: comma groups of three
+        ("123,456,789", false),      // a comma-grouped number, not a TIN
+        ("(123) 456-7890", false),   // a phone number: 3-3-4
         ("-123456789-", true),
     ];
     for (text, want) in cases {
@@ -242,17 +250,20 @@ fn has_full_tin_unicode_digits() {
     assert!(has_full_tin("١٢٣٤٥٦٧٨٩")); // Arabic-Indic digits
 }
 
-/// Oracle: `has_full_tin` doc (P3 review round 2): past ASCII, every form
+/// Oracle: `has_full_tin` doc (P3 review round 2; P7): past ASCII, every
-/// of the five separators Unicode has separates (every Zs space, every
+/// form of the five separators Unicode has separates (every Zs space, every
 /// Dash character, the compatibility forms of the dot, slash and
 /// underscore; `docs/spec/sources/unicode-tin-text.md`, Unicode 17.0),
 /// and invisible characters (Default_Ignorable_Code_Point) don't count
-/// toward the gap. A TIN typed in fullwidth forms, Arabic-Indic digits,
+/// toward the gap. A run of spaces counts once, however long, and a comma
-/// with an en dash, a non-breaking hyphen, a Hebrew maqaf, any space, or
+/// or a parenthesis separates a TIN's 3-2-4 or 2-7 groups (ASCII only).
-/// with right-to-left marks around its hyphens is still a full TIN; line
+/// A TIN typed in fullwidth forms, Arabic-Indic digits, with an en dash, a
-/// breaks, commas and a fourth visible separator still end a run.
+/// non-breaking hyphen, a Hebrew maqaf, any space, a comma or a
-/// (Before, "１２３－４５－６７８９" passed: the fullwidth hyphen-minus wasn't
+/// parenthesis, or with right-to-left marks around its hyphens is still a
-/// a separator, nor the ideographic or thin space.)
+/// full TIN; line breaks, commas in an ungrouped run, tabs and a fourth
+/// other visible separator still end a run. (Before P3, "１２３－４５－６７８９"
+/// passed: the fullwidth hyphen-minus wasn't a separator, nor the
+/// ideographic or thin space.)
 #[test]
 fn has_full_tin_reads_every_script_and_separator() {
     let tins = [
@@ -264,21 +275,26 @@ fn has_full_tin_reads_every_script_and_separator() {
         "\u{661}\u{662}\u{663}-\u{664}\u{665}-\u{666}\u{667}\u{668}\u{669}",
         "\u{6F1}\u{6F2}\u{6F3}\u{6F4}\u{6F5}\u{6F6}\u{6F7}\u{6F8}\u{6F9}",
         "\u{967}\u{968}\u{969}\u{2013}\u{96A}\u{96B}\u{2013}\u{96C}\u{96D}\u{96E}\u{96F}",
-        "291\u{2013}12\u{2013}2846",     // en dash
+        "291\u{2013}12\u{2013}2846",                 // en dash
-        "291\u{2011}12\u{2011}2846",     // non-breaking hyphen
+        "291\u{2011}12\u{2011}2846",                 // non-breaking hyphen
-        "291\u{2012}12\u{2012}2846",     // figure dash
+        "291\u{2012}12\u{2012}2846",                 // figure dash
-        "291\u{2015}12\u{2015}2846",     // horizontal bar
+        "291\u{2015}12\u{2015}2846",                 // horizontal bar
-        "291\u{5BE}12\u{5BE}2846",       // Hebrew maqaf
+        "291\u{5BE}12\u{5BE}2846",                   // Hebrew maqaf
-        "291\u{301C}12\u{301C}2846",     // wave dash
+        "291\u{301C}12\u{301C}2846",                 // wave dash
-        "291\u{A0}12\u{A0}2846",         // no-break space
+        "291\u{A0}12\u{A0}2846",                     // no-break space
-        "291\u{3000}12\u{3000}2846",     // ideographic space
+        "291\u{3000}12\u{3000}2846",                 // ideographic space
-        "291\u{2009}12\u{2009}2846",     // thin space
+        "291\u{2009}12\u{2009}2846",                 // thin space
-        "291\u{202F}12\u{202F}2846",     // narrow no-break space
+        "291\u{202F}12\u{202F}2846",                 // narrow no-break space
-        "291 \u{2014} 12 \u{2014} 2846", // three: space, em dash, space
+        "291 \u{2014} 12 \u{2014} 2846",             // three: space, em dash, space
-        "291\u{FF0E}12\u{FF0E}2846",     // fullwidth full stop
+        "12\u{3000}\u{3000}\u{3000}\u{3000}3456789", // a run of spaces counts once
-        "291\u{2024}12\u{2024}2846",     // one dot leader
+        "291\u{FF0E}12\u{FF0E}2846",                 // fullwidth full stop
-        "291\u{FF0F}12\u{FF0F}2846",     // fullwidth solidus
+        "291\u{2024}12\u{2024}2846",                 // one dot leader
-        "291\u{FF3F}12\u{FF3F}2846",     // fullwidth low line
+        "291\u{FF0F}12\u{FF0F}2846",                 // fullwidth solidus
+        "291\u{FF3F}12\u{FF3F}2846",                 // fullwidth low line
+        "291,12,2846",                               // commas between the groups
+        "291, 12, 2846",                             // a comma and a space
+        "(291) 12-2846",                             // parentheses around a group
+        "12,3456789",                                // an EIN's groups
         // Zero-width spaces between every digit.
         "2\u{200B}9\u{200B}1\u{200B}1\u{200B}2\u{200B}2\u{200B}8\u{200B}4\u{200B}6",
         "291\u{200F}-\u{200F}12\u{200F}-\u{200F}2846", // right-to-left marks around hyphens
@@ -308,24 +324,25 @@ fn has_full_tin_reads_every_script_and_separator() {
         "•••\u{200F}-\u{200F}••-•••• ok"
     );
     let not = [
-        "12\u{3000}\u{3000}\u{3000}\u{3000}3456789", // four visible separators
+        "12 \u{2014} \u{200B}-3456789", // four visible, an invisible one among them
-        "12 \u{2014} \u{200B}-3456789",              // four, an invisible one among them
+        "291\u{2028}12\u{2028}2846",    // line separators end a run
-        "291\u{2028}12\u{2028}2846",                 // line separators end a run
+        "291\u{2029}12\u{2029}2846",    // paragraph separators
-        "291\u{2029}12\u{2029}2846",                 // paragraph separators
+        "291\u{85}12\u{85}2846",        // next line
-        "291\u{85}12\u{85}2846",                     // next line
+        "291\u{FF0C}12\u{FF0C}2846",    // fullwidth commas are not a group's comma
-        "291\u{FF0C}12\u{FF0C}2846",                 // fullwidth commas
+        "291\u{3001}12\u{3001}2846",    // ideographic commas either
-        "291\u{3001}12\u{3001}2846",                 // ideographic commas
+        "1,234,567.89",                 // an amount: comma groups of three
+        "(123) 456-7890",               // a phone number: 3-3-4
         "\u{FF11}\u{FF12}\u{FF13}\u{FF14}\u{FF15}\u{FF16}\u{FF17}\u{FF18}", // eight digits
-        "291\u{200B}12284",                          // eight, invisibly joined
+        "291\u{200B}12284",             // eight, invisibly joined
     ];
     for t in not {
         assert!(!has_full_tin(t), "has_full_tin({t:?})");
     }
 }
 
-/// The reading before P3 review round 2, the oracle for ASCII: past it,
+/// The reading before P3 review round 2, the oracle for what P7 must
-/// only the no-break space and a few dashes separated, and every
+/// never narrow: P7 only widens what reads as a TIN (commas and
-/// separator counted.
+/// parentheses between a TIN's groups, any run of spaces counting once).
 fn has_full_tin_before(text: &str) -> bool {
     let sep = |c: char| {
         matches!(
@@ -366,15 +383,13 @@ fn has_full_tin_before(text: &str) -> bool {
 proptest! {
     #![proptest_config(ProptestConfig::with_cases(2000))]
 
-    /// `has_full_tin` reads ASCII exactly as it did (the fix only widens
+    /// What `has_full_tin` found before P7 it still finds: the fix only
-    /// what separates past ASCII), and what it found before in any text
+    /// widens the reading (commas and parentheses between a TIN's groups,
-    /// it still finds: it only ever finds more.
+    /// any run of spaces counting once), never narrows it.
     #[test]
-    fn has_full_tin_keeps_its_ascii_reading(
+    fn has_full_tin_never_forgets_a_tin(
-        ascii in "[0-9 ./_\t\n\r,:a-]{0,40}",
         any in "[0-9 ./_\t\n,a\u{A0}\u{2010}-\u{2015}\u{2212}\u{FF0D}\u{3000}\u{200B}\u{200F}\u{FF10}-\u{FF19}\u{660}-\u{669}\u{2028}-]{0,40}",
     ) {
-        prop_assert_eq!(has_full_tin(&ascii), has_full_tin_before(&ascii), "{:?}", ascii);
         if has_full_tin_before(&any) {
             prop_assert!(has_full_tin(&any), "{:?}", any);
         }