wip: pause checkpoint — P6 feature complete, gate interrupted (not for merge)

crates/ledger-import/Cargo.toml

@@ -18,6 +18,17 @@ typst = "0.15.1"
 serde_json = "1"
 typst-layout = "0.15.1"
 qrcode = { version = "0.14.1", default-features = false, features = ["svg"] }
+# Extraction (extract.rs, plan P6): a PDF's text, a page's pixels, an
+# XLSX's cells. hayro (the renderer) and hayro-interpret (the text
+# interpreter) are pure Rust and already the viewer's PDF reader, so the
+# server and the viewer read a receipt through one implementation rather
+# than two; `zip` and `quick-xml` read a spreadsheet's XML. None of them
+# touches money: a number is kept as the decimal string the XML holds.
+hayro = { version = "0.8", features = ["embed-fonts"] }
+hayro-interpret = "0.8"
+kurbo = "0.13"
+zip = { version = "9.0.1", default-features = false, features = ["deflate"] }
+quick-xml = "0.38"
 # BSD-3-Clause: the IIF's transliteration to ASCII (iif.rs, rules.md B-34).
 # Pinned exactly: a new table would change what a CPA's file says.
 deunicode = { version = "=1.6.0", default-features = false }
@@ -27,8 +38,4 @@ workspace = true
 
 [dev-dependencies]
 proptest = "1"
-# The viewer's PDF reader (ledger-gui's hayro): reads back the text a PDF
-# draws, glyph by glyph, in the tests.
-hayro-interpret = "0.8"
-kurbo = "0.13"
 rqrr = { version = "0.11.0", default-features = false }

crates/ledger-import/src/extract.rs

@@ -0,0 +1,625 @@
+//! Reading the words and pages out of the files the books keep: a PDF's
+//! text and a page as a picture, an XLSX's cells as exact decimal strings,
+//! and plain text as it is (plan P6 step 1, D-32, D-53).
+//!
+//! ```text
+//! GOAL: one extractor, in this process, that the server's filing scan,
+//!       the journal's extraction record and the viewer all read a file
+//!       through, so a receipt's text never depends on which door it came
+//!       in; a reader failure is an error, never a panic out of the
+//!       server.
+//! SOURCE: plan/served.md P6 step 1 (D-32: no sandbox; D-53: in-process,
+//!         no new crate); IRS Pub. 583 (records must be readable); D-15
+//!         (no OCR: an image-only receipt is its page pictures).
+//! INVARIANTS:
+//!   1. Deterministic: the same bytes give the same text and the same PNG
+//!      bytes, every run.
+//!   2. Text is read, never guessed: an XLSX cell's number is the decimal
+//!      string the sheet XML holds, never a binary float; a PDF's text is
+//!      its glyphs' ToUnicode text, page by page, line by line.
+//!   3. Caps: at most 25 MiB of input, 30 pages, 2 MiB of text. Past the
+//!      input cap nothing is read; past the page cap the rest is not
+//!      read; past the text cap the text is cut at a character boundary.
+//!   4. A panic inside the PDF reader is caught here and reported as
+//!      [`ExtractError::ReaderPanicked`]: the process is not isolated
+//!      (D-32, and the plan says so).
+//! ```
+
+use std::io::{Cursor, Read};
+use thiserror::Error;
+
+/// The largest file extraction reads: 25 MiB (plan P6 step 1).
+pub const MAX_BYTES: usize = 25 * 1024 * 1024;
+
+/// The most pages of a PDF extraction reads.
+pub const MAX_PAGES: usize = 30;
+
+/// The most text one extraction keeps: 2 MiB.
+pub const MAX_TEXT_BYTES: usize = 2 * 1024 * 1024;
+
+/// What read a file's text, as the extraction record names it.
+pub const EXTRACTOR: &str = "wanderbooks";
+
+/// That extractor's version: a new one journals a new extraction record,
+/// never replaces an old one (plan P6 step 2).
+pub const VERSION: &str = env!("CARGO_PKG_VERSION");
+
+/// What reading a file found (plan P6 step 2's `status`).
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum ExtractStatus {
+    /// Text was read.
+    Ok,
+    /// There is no text to read (an image is its page pictures, D-15).
+    None,
+    /// Reading was tried and failed.
+    Failed,
+}
+
+impl ExtractStatus {
+    /// Its wire name, as the extraction record journals it.
+    pub fn key(self) -> &'static str {
+        match self {
+            ExtractStatus::Ok => "ok",
+            ExtractStatus::None => "none",
+            ExtractStatus::Failed => "failed",
+        }
+    }
+}
+
+/// What [`extract`] found: the text (when any), the file's page count, and
+/// whether there was text to read.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct Extracted {
+    /// The text, capped at [`MAX_TEXT_BYTES`].
+    pub text: Option<String>,
+    /// How many pages the file has (0 when it has none).
+    pub pages: u32,
+    /// What was found.
+    pub status: ExtractStatus,
+}
+
+/// Why a file's text or page could not be read.
+#[derive(Debug, Error)]
+pub enum ExtractError {
+    /// The file is past [`MAX_BYTES`].
+    #[error("the file is {size} bytes: extraction reads at most {MAX_BYTES} bytes")]
+    TooLarge {
+        /// The file's size.
+        size: usize,
+    },
+    /// The bytes are not a PDF this reader can read.
+    #[error("not a readable PDF: {0}")]
+    Pdf(String),
+    /// The bytes are not a spreadsheet this reader can read.
+    #[error("not a readable spreadsheet: {0}")]
+    Xlsx(String),
+    /// The page asked for is past the document's last.
+    #[error("page {page} is past the document's {pages} page(s)")]
+    PageOutOfRange {
+        /// The page asked for (1-based).
+        page: usize,
+        /// How many pages the document has.
+        pages: usize,
+    },
+    /// The page size asked for is zero.
+    #[error("a page image needs a size of at least 1 pixel")]
+    NoPixels,
+    /// The PDF reader panicked on the file (INVARIANT 4).
+    #[error("the PDF reader stopped on that file")]
+    ReaderPanicked,
+}
+
+/// Run `read`, turning a panic into [`ExtractError::ReaderPanicked`]
+/// (INVARIANT 4).
+fn caught<T>(read: impl FnOnce() -> Result<T, ExtractError>) -> Result<T, ExtractError> {
+    std::panic::catch_unwind(std::panic::AssertUnwindSafe(read))
+        .unwrap_or(Err(ExtractError::ReaderPanicked))
+}
+
+/// The text a PDF draws: each glyph's text, left to right, one `String`
+/// per glyph, one `Vec` per line, from the top of each page down, page
+/// after page (INVARIANT 2). At most [`MAX_PAGES`] pages are read.
+pub fn pdf_lines(bytes: &[u8]) -> Result<Vec<Vec<String>>, ExtractError> {
+    if bytes.len() > MAX_BYTES {
+        return Err(ExtractError::TooLarge { size: bytes.len() });
+    }
+    caught(|| read_pdf_lines(bytes))
+}
+
+/// The text a PDF draws, its glyphs joined into lines and its pages into
+/// one text (INVARIANTS 1 and 2), capped at [`MAX_TEXT_BYTES`].
+pub fn pdf_text(bytes: &[u8]) -> Result<String, ExtractError> {
+    let lines = pdf_lines(bytes)?;
+    let text = lines
+        .iter()
+        .map(|l| l.concat())
+        .collect::<Vec<_>>()
+        .join("\n");
+    Ok(cap_text(text))
+}
+
+/// How many pages a PDF has.
+pub fn page_count(bytes: &[u8]) -> Result<usize, ExtractError> {
+    if bytes.len() > MAX_BYTES {
+        return Err(ExtractError::TooLarge { size: bytes.len() });
+    }
+    caught(|| Ok(read_pdf(bytes)?.pages().len()))
+}
+
+/// One page of a PDF as a PNG whose longest side is at most `max_side_px`
+/// pixels (`page` is 0-based; `max_side_px` of 0 is refused). The same
+/// bytes, page and size give the same PNG bytes (INVARIANT 1).
+pub fn page_png(bytes: &[u8], page: usize, max_side_px: u32) -> Result<Vec<u8>, ExtractError> {
+    if bytes.len() > MAX_BYTES {
+        return Err(ExtractError::TooLarge { size: bytes.len() });
+    }
+    if max_side_px == 0 {
+        return Err(ExtractError::NoPixels);
+    }
+    caught(|| read_page_png(bytes, page, max_side_px))
+}
+
+/// The bytes as text, if they are text the books read (INVARIANT 2): a
+/// PDF's text, an XLSX's cells, UTF-16 with its byte-order mark, UTF-8, or,
+/// for a text type or name, whatever they decode to. Text with NUL bytes
+/// and no byte-order mark (UTF-16 written without one is valid UTF-8, its
+/// digits split by NULs) is read every way it can be (as it is, as UTF-16
+/// both ways round, and with its NULs taken out), each reading on a line of
+/// its own: so is bytes that aren't UTF-8 but are NUL-interleaved as UTF-16
+/// is. At most [`MAX_TEXT_BYTES`] is returned.
+pub fn text_of(bytes: &[u8], mime_type: &str, file_name: &str) -> Option<String> {
+    extract(bytes, mime_type, file_name).text
+}
+
+/// Read a file the way the books record it (plan P6 step 2): a PDF's text
+/// and pages, an XLSX's cells, plain text as it is; an image has none. A
+/// reader failure is [`ExtractStatus::Failed`], never a panic (INVARIANT 4).
+pub fn extract(bytes: &[u8], mime_type: &str, file_name: &str) -> Extracted {
+    if bytes.len() > MAX_BYTES {
+        return Extracted {
+            text: None,
+            pages: 0,
+            status: ExtractStatus::Failed,
+        };
+    }
+    let mime = mime_type.trim().to_ascii_lowercase();
+    let ext = std::path::Path::new(file_name)
+        .extension()
+        .and_then(|e| e.to_str())
+        .map(str::to_ascii_lowercase)
+        .unwrap_or_default();
+    let magic_pdf = bytes.starts_with(b"%PDF");
+    if mime == "application/pdf" || magic_pdf {
+        return match pdf_text(bytes) {
+            Ok(text) => {
+                let pages = page_count(bytes).unwrap_or(0).min(MAX_PAGES) as u32;
+                found(Some(text), pages)
+            }
+            // A file CALLED a PDF whose bytes are text (a text file named
+            // .pdf): read the text, so the TIN scan and the reader see what
+            // it says (B-26: the scan fails closed). One with the PDF magic
+            // that won't read is a failed extraction.
+            Err(_) if !magic_pdf => match plain_text(bytes, &mime, &ext) {
+                Some(text) => found(Some(text), 1),
+                None => Extracted {
+                    text: None,
+                    pages: 0,
+                    status: ExtractStatus::Failed,
+                },
+            },
+            Err(_) => Extracted {
+                text: None,
+                pages: 0,
+                status: ExtractStatus::Failed,
+            },
+        };
+    }
+    let xlsx = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
+    if mime == xlsx || ext == "xlsx" || ext == "xlsm" {
+        return match xlsx_text(bytes) {
+            Ok(text) => found(Some(text), 1),
+            Err(_) => Extracted {
+                text: None,
+                pages: 0,
+                status: ExtractStatus::Failed,
+            },
+        };
+    }
+    match plain_text(bytes, &mime, &ext) {
+        Some(text) => found(Some(text), 1),
+        None => Extracted {
+            text: None,
+            pages: 0,
+            status: ExtractStatus::None,
+        },
+    }
+}
+
+/// An extraction's answer from the text read: `none` when it is blank.
+fn found(text: Option<String>, pages: u32) -> Extracted {
+    match text {
+        Some(t) if t.trim().is_empty() => Extracted {
+            text: None,
+            pages,
+            status: ExtractStatus::None,
+        },
+        Some(t) => Extracted {
+            text: Some(t),
+            pages,
+            status: ExtractStatus::Ok,
+        },
+        None => Extracted {
+            text: None,
+            pages,
+            status: ExtractStatus::None,
+        },
+    }
+}
+
+/// The plain-text reading of bytes (the second half of [`text_of`]).
+fn plain_text(bytes: &[u8], mime: &str, ext: &str) -> Option<String> {
+    let utf16 = |body: &[u8], le: bool| {
+        let units: Vec<u16> = body
+            .as_chunks::<2>()
+            .0
+            .iter()
+            .map(|&c| {
+                if le {
+                    u16::from_le_bytes(c)
+                } else {
+                    u16::from_be_bytes(c)
+                }
+            })
+            .collect();
+        String::from_utf16_lossy(&units)
+    };
+    let text = match bytes {
+        [0xFF, 0xFE, body @ ..] => utf16(body, true),
+        [0xFE, 0xFF, body @ ..] => utf16(body, false),
+        _ => {
+            let plain = match std::str::from_utf8(bytes) {
+                Ok(t) => Some(t.to_string()),
+                Err(_) if text_like(mime, ext) => Some(String::from_utf8_lossy(bytes).into_owned()),
+                Err(_) => None,
+            };
+            if !bytes.contains(&0) || plain.is_none() && !nul_interleaved(bytes) {
+                return plain.map(cap_text);
+            }
+            let without_nuls: Vec<u8> = bytes.iter().copied().filter(|&b| b != 0).collect();
+            let readings = [
+                plain.unwrap_or_default(),
+                utf16(bytes, true),
+                utf16(bytes, false),
+                String::from_utf8_lossy(&without_nuls).into_owned(),
+            ];
+            readings.join("\n")
+        }
+    };
+    Some(cap_text(text))
+}
+
+/// [`text_of`]'s answer capped at [`MAX_TEXT_BYTES`], cut at a character
+/// boundary.
+fn cap_text(mut text: String) -> String {
+    if text.len() > MAX_TEXT_BYTES {
+        let mut cut = MAX_TEXT_BYTES;
+        while !text.is_char_boundary(cut) {
+            cut -= 1;
+        }
+        text.truncate(cut);
+    }
+    text
+}
+
+/// Whether a file of this type or name is text: csv, tsv, txt, json, iif,
+/// xml, html, md and their kin.
+fn text_like(mime: &str, ext: &str) -> bool {
+    mime.starts_with("text/")
+        || [
+            "json",
+            "xml",
+            "csv",
+            "x-iif",
+            "javascript",
+            "x-yaml",
+            "yaml",
+        ]
+        .iter()
+        .any(|t| mime.ends_with(&format!("/{t}")) || mime.ends_with(&format!("+{t}")))
+        || [
+            "csv", "tsv", "txt", "text", "json", "iif", "xml", "html", "htm", "md", "markdown",
+            "yaml", "yml", "tab", "prn", "log", "ofx", "qfx", "qif",
+        ]
+        .contains(&ext)
+}
+
+/// Whether `bytes` look like UTF-16 without a byte-order mark: of their
+/// byte pairs, at least half have a NUL on one side (the high byte of a
+/// Latin character) and at most an eighth on the other.
+#[doc(hidden)]
+pub fn nul_interleaved(bytes: &[u8]) -> bool {
+    let pairs = bytes.as_chunks::<2>().0;
+    let nuls = |side: usize| pairs.iter().filter(|p| p[side] == 0).count();
+    let (first, second) = (nuls(0), nuls(1));
+    let n = pairs.len();
+    n > 0 && (2 * first >= n && 8 * second <= n || 2 * second >= n && 8 * first <= n)
+}
+
+// ---------------------------------------------------------------------------
+// PDF
+// ---------------------------------------------------------------------------
+
+type Pdf = hayro_interpret::hayro_syntax::Pdf;
+
+/// [`Pdf::new`] with its error as ours.
+fn read_pdf(bytes: &[u8]) -> Result<Pdf, ExtractError> {
+    Pdf::new(bytes.to_vec()).map_err(|e| ExtractError::Pdf(format!("{e:?}")))
+}
+
+fn read_pdf_lines(bytes: &[u8]) -> Result<Vec<Vec<String>>, ExtractError> {
+    use hayro_interpret::font::GlyphRun;
+    use hayro_interpret::hayro_cmap::BfString;
+    use hayro_interpret::{
+        BlendMode, ClipPath, Context, Device, DrawMode, DrawProps, Image, ImageDrawProps,
+        InterpreterCache, InterpreterSettings, SoftMask, interpret_page,
+    };
+    use kurbo::{Affine, BezPath, Point, Rect};
+    /// Each glyph drawn: its baseline (hundredths of a point), x, and text.
+    struct Glyphs(Vec<(i64, f64, String)>);
+    impl<'a> Device<'a> for Glyphs {
+        fn draw_path(&mut self, _: &BezPath, _: DrawProps<'a>, _: &DrawMode) {}
+        fn push_clip_path(&mut self, _: &ClipPath) {}
+        fn push_transparency_group(&mut self, _: f32, _: Option<SoftMask<'a>>, _: BlendMode) {}
+        fn draw_glyph_run(&mut self, run: &GlyphRun<'_, 'a>, props: DrawProps<'a>, _: &DrawMode) {
+            for g in run.glyphs() {
+                let at = (props.transform * g.transform()) * Point::ZERO;
+                let text = match g.as_unicode() {
+                    Some(BfString::Char(c)) => c.to_string(),
+                    Some(BfString::String(s)) => s,
+                    // A mark or a ligature's second glyph: its text is with
+                    // the cluster's first.
+                    None => String::new(),
+                };
+                self.0.push(((at.y * 100.0).round() as i64, at.x, text));
+            }
+        }
+        fn draw_image(&mut self, _: Image<'a, '_>, _: ImageDrawProps<'a>) {}
+        fn pop_clip(&mut self) {}
+        fn pop_transparency_group(&mut self) {}
+    }
+    let pdf = read_pdf(bytes)?;
+    let cache = InterpreterCache::new();
+    let mut out = Vec::new();
+    for page in pdf.pages().iter().take(MAX_PAGES) {
+        let mut ctx = Context::new(
+            Affine::IDENTITY,
+            Rect::new(0.0, 0.0, 1.0, 1.0),
+            &cache,
+            pdf.xref(),
+            InterpreterSettings::default(),
+        );
+        let mut glyphs = Glyphs(Vec::new());
+        interpret_page(page, &mut ctx, &mut glyphs);
+        // PDF space runs up the page: the highest baseline first.
+        let mut lines: std::collections::BTreeMap<std::cmp::Reverse<i64>, Vec<(f64, String)>> =
+            std::collections::BTreeMap::new();
+        for (y, x, t) in glyphs.0 {
+            lines.entry(std::cmp::Reverse(y)).or_default().push((x, t));
+        }
+        for (_, mut line) in lines {
+            line.sort_by(|a, b| a.0.total_cmp(&b.0));
+            out.push(line.into_iter().map(|(_, t)| t).collect());
+        }
+    }
+    Ok(out)
+}
+
+fn read_page_png(bytes: &[u8], page: usize, max_side_px: u32) -> Result<Vec<u8>, ExtractError> {
+    use hayro::hayro_interpret::InterpreterSettings;
+    use hayro::vello_cpu::color::palette::css::WHITE;
+    use hayro::{PixmapSettings, RenderCache, RenderSettings, render};
+    let pdf = read_pdf(bytes)?;
+    let pages = pdf.pages();
+    let found = pages.get(page).ok_or(ExtractError::PageOutOfRange {
+        page: page + 1,
+        pages: pages.len(),
+    })?;
+    let (w, h) = found.render_dimensions();
+    // The page's longest side at most `max_side_px`; never zero.
+    let scale = max_side_px as f32 / w.max(h).max(1.0);
+    let scale = scale.max(1.0 / w.max(h).max(1.0));
+    let px = PixmapSettings {
+        x_scale: scale,
+        y_scale: scale,
+        bg_color: WHITE,
+    };
+    let pixmap = render(
+        found,
+        &RenderCache::new(),
+        &InterpreterSettings::default(),
+        &RenderSettings::default(),
+        &px,
+    );
+    pixmap
+        .into_png()
+        .map_err(|e| ExtractError::Pdf(format!("{e:?}")))
+}
+
+// ---------------------------------------------------------------------------
+// XLSX
+// ---------------------------------------------------------------------------
+
+/// An XLSX's cells as text: one line per row, one cell per column,
+/// separated by tabs; a number is the exact decimal string the sheet XML
+/// holds, never a binary float (INVARIANT 2).
+pub fn xlsx_text(bytes: &[u8]) -> Result<String, ExtractError> {
+    if bytes.len() > MAX_BYTES {
+        return Err(ExtractError::TooLarge { size: bytes.len() });
+    }
+    let mut zip =
+        zip::ZipArchive::new(Cursor::new(bytes)).map_err(|e| ExtractError::Xlsx(e.to_string()))?;
+    // The workbook's shared strings, in index order.
+    let shared = match read_zip(&mut zip, "xl/sharedStrings.xml") {
+        Some(xml) => shared_strings(&xml)?,
+        None => Vec::new(),
+    };
+    let names: Vec<String> = zip
+        .file_names()
+        .filter_map(|n| n.ok().map(|n| n.into_owned()))
+        .filter(|n| n.starts_with("xl/worksheets/sheet") && n.ends_with(".xml"))
+        .collect();
+    let mut sheets = names;
+    sheets.sort();
+    let mut out = String::new();
+    for name in sheets {
+        let Some(xml) = read_zip(&mut zip, &name) else {
+            continue;
+        };
+        let text = sheet_cells(&xml, &shared)?;
+        if !text.is_empty() {
+            if !out.is_empty() {
+                out.push('\n');
+            }
+            out.push_str(&text);
+        }
+    }
+    Ok(cap_text(out))
+}
+
+/// A zip member's text, if it is there.
+fn read_zip(zip: &mut zip::ZipArchive<Cursor<&[u8]>>, name: &str) -> Option<String> {
+    let mut file = zip.by_name(name).ok()?;
+    let mut bytes = Vec::new();
+    file.read_to_end(&mut bytes).ok()?;
+    Some(String::from_utf8_lossy(&bytes).into_owned())
+}
+
+/// The text of each `<si>` in `xl/sharedStrings.xml`, rich-text runs
+/// concatenated.
+fn shared_strings(xml: &str) -> Result<Vec<String>, ExtractError> {
+    let mut reader = quick_xml::Reader::from_str(xml);
+    let mut out = Vec::new();
+    let mut current: Option<String> = None;
+    let mut in_text = false;
+    loop {
+        match reader.read_event() {
+            Ok(quick_xml::events::Event::Start(e)) if e.name().as_ref() == b"si" => {
+                current = Some(String::new());
+            }
+            Ok(quick_xml::events::Event::End(e)) if e.name().as_ref() == b"si" => {
+                out.push(current.take().unwrap_or_default());
+            }
+            Ok(quick_xml::events::Event::Start(e)) if e.name().as_ref() == b"t" => in_text = true,
+            Ok(quick_xml::events::Event::End(e)) if e.name().as_ref() == b"t" => in_text = false,
+            Ok(quick_xml::events::Event::Text(t)) if in_text => {
+                if let Some(s) = &mut current {
+                    s.push_str(&t.decode().map_err(|e| ExtractError::Xlsx(e.to_string()))?);
+                }
+            }
+            Ok(quick_xml::events::Event::Eof) => break,
+            Ok(_) => {}
+            Err(e) => return Err(ExtractError::Xlsx(e.to_string())),
+        }
+    }
+    Ok(out)
+}
+
+/// One worksheet's cells as tab-separated rows.
+fn sheet_cells(xml: &str, shared: &[String]) -> Result<String, ExtractError> {
+    use quick_xml::events::Event;
+    let mut reader = quick_xml::Reader::from_str(xml);
+    let mut rows: Vec<Vec<String>> = Vec::new();
+    let mut row: Vec<String> = Vec::new();
+    let mut cell_type = String::new();
+    let mut in_value = false;
+    let mut value = String::new();
+    let mut in_inline = false;
+    let mut inline = String::new();
+    let mut in_inline_text = false;
+    loop {
+        match reader.read_event() {
+            Ok(Event::Start(e)) => match e.name().as_ref() {
+                b"row" => row = Vec::new(),
+                b"c" => {
+                    cell_type.clear();
+                    value.clear();
+                    inline.clear();
+                    for a in e.attributes().flatten() {
+                        if a.key.as_ref() == b"t" {
+                            cell_type = String::from_utf8_lossy(&a.value).into_owned();
+                        }
+                    }
+                }
+                b"v" => {
+                    in_value = true;
+                    value.clear();
+                }
+                b"is" => in_inline = true,
+                b"t" if in_inline => in_inline_text = true,
+                _ => {}
+            },
+            Ok(Event::End(e)) => match e.name().as_ref() {
+                b"row" => rows.push(std::mem::take(&mut row)),
+                b"c" => {
+                    let text = match cell_type.as_str() {
+                        "s" => value
+                            .parse::<usize>()
+                            .ok()
+                            .and_then(|i| shared.get(i).cloned())
+                            .unwrap_or_default(),
+                        "inlineStr" => inline.clone(),
+                        "b" => match value.as_str() {
+                            "1" => "TRUE".into(),
+                            _ => "FALSE".into(),
+                        },
+                        _ => value.clone(),
+                    };
+                    row.push(text);
+                }
+                b"v" => in_value = false,
+                b"is" => in_inline = false,
+                b"t" if in_inline => in_inline_text = false,
+                _ => {}
+            },
+            Ok(Event::Text(t)) => {
+                let text = t.decode().map_err(|e| ExtractError::Xlsx(e.to_string()))?;
+                if in_inline_text {
+                    inline.push_str(&text);
+                } else if in_value {
+                    value.push_str(&text);
+                }
+            }
+            Ok(Event::Eof) => break,
+            Ok(_) => {}
+            Err(e) => return Err(ExtractError::Xlsx(e.to_string())),
+        }
+    }
+    Ok(rows
+        .iter()
+        .map(|r| r.join("\t"))
+        .collect::<Vec<_>>()
+        .join("\n"))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// INVARIANT 4: a panic inside the reader is caught, never fatal.
+    #[test]
+    fn a_reader_panic_is_reported_not_raised() {
+        let got = caught::<()>(|| panic!("the reader broke"));
+        assert!(matches!(got, Err(ExtractError::ReaderPanicked)));
+    }
+
+    /// INVARIANT 3: a file past the input cap is not read.
+    #[test]
+    fn too_large_is_refused_without_reading() {
+        let bytes = vec![0u8; MAX_BYTES + 1];
+        assert!(matches!(
+            pdf_text(&bytes),
+            Err(ExtractError::TooLarge { .. })
+        ));
+        assert!(text_of(&bytes, "text/plain", "big.txt").is_none());
+    }
+}

crates/ledger-import/src/lib.rs

@@ -16,6 +16,7 @@
 pub mod asset_register;
 pub mod bank_csv;
 pub mod chart_csv;
+pub mod extract;
 pub mod fonts;
 pub mod iif;
 pub mod invoice_pdf;
@@ -28,6 +29,7 @@ pub mod xlsx;
 pub use asset_register::{asset_register_pdf, asset_register_xlsx};
 pub use bank_csv::{BankCsvError, BankTransaction, DateOrder, parse_bank_csv, parse_bank_csv_with};
 pub use chart_csv::{ChartRow, parse_chart_csv};
+pub use extract::{ExtractError, page_count, page_png, pdf_lines, pdf_text, text_of, xlsx_text};
 pub use iif::{
     Iif, IifAccount, IifChange, IifClass, IifError, IifField, LeftOut, MEMO_MAX, NAME_MAX,
     NUMBER_MAX, QbAccountType, iif_accounts, iif_classes, journal_to_iif,

crates/ledger-import/tests/extract.rs

@@ -0,0 +1,126 @@
+//! The one extractor (plan P6 step 1): a PDF's text and pages, and an
+//! XLSX's cells as exact decimal strings.
+//!
+//! Expected behavior comes from `crates/ledger-import/src/extract.rs`'s
+//! INVARIANTS, which come from plan/served.md P6 step 1 and D-15, D-32 and
+//! D-53.
+
+use std::io::{Cursor, Write};
+
+use ledger_import::extract::{ExtractError, MAX_BYTES, page_png, pdf_text, text_of, xlsx_text};
+use ledger_import::report::{ReportMeta, Row, RowKind, statement};
+use ledger_import::statement_pdf;
+use proptest::prelude::*;
+use rust_decimal::Decimal;
+
+fn a_pdf() -> Vec<u8> {
+    let st = statement(
+        "Receipt",
+        "March 2026".to_string(),
+        &["Total"],
+        vec![Row {
+            kind: RowKind::Line,
+            label: "Rent".into(),
+            amounts: vec![Some(Decimal::new(10000, 2))],
+            currency: true,
+            rule_above: false,
+        }],
+    );
+    let meta = ReportMeta {
+        company: "Fernhill Labs LLC".into(),
+        basis: "Accrual".into(),
+        prepared_on: chrono::NaiveDate::from_ymd_opt(2026, 3, 2).unwrap(),
+        journal_seq: 1,
+    };
+    statement_pdf(&st, &meta).unwrap()
+}
+
+/// An XLSX holding one sheet whose cells are `values` (numbers as written,
+/// not through a float).
+fn xlsx_of(values: &[String]) -> Vec<u8> {
+    let mut writer = zip::ZipWriter::new(Cursor::new(Vec::new()));
+    let options = zip::write::SimpleFileOptions::default();
+    writer
+        .start_file("xl/worksheets/sheet1.xml", options)
+        .unwrap();
+    let cells: String = values
+        .iter()
+        .enumerate()
+        .map(|(i, v)| format!(r#"<c r="A{}"><v>{v}</v></c>"#, i + 1))
+        .collect();
+    writer
+        .write_all(
+            format!(
+                r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData><row r="1">{cells}</row></sheetData></worksheet>"#
+            )
+            .as_bytes(),
+        )
+        .unwrap();
+    writer.finish().unwrap().into_inner()
+}
+
+/// INVARIANT 1: the same bytes give the same text and the same PNG bytes,
+/// every run.
+#[test]
+fn extraction_deterministic() {
+    let pdf = a_pdf();
+    let first = pdf_text(&pdf).unwrap();
+    assert_eq!(first, pdf_text(&pdf).unwrap(), "the text");
+    assert!(first.contains("Rent"), "{first}");
+    let first_png = page_png(&pdf, 0, 600).unwrap();
+    assert_eq!(first_png, page_png(&pdf, 0, 600).unwrap(), "the page");
+    assert!(first_png.starts_with(b"\x89PNG\r\n\x1a\n"), "a PNG");
+    // A plain text file reads back exactly; an image has no text (D-15).
+    assert_eq!(
+        text_of(b"date,amount\n2026-03-01,12.50\n", "text/csv", "bank.csv").as_deref(),
+        Some("date,amount\n2026-03-01,12.50\n")
+    );
+    assert_eq!(text_of(b"\x89PNG\r\n\x1a\n", "image/png", "r.png"), None);
+}
+
+// INVARIANT 2: an XLSX cell's number is the exact decimal string the
+// sheet XML holds, never a binary float.
+proptest! {
+    #[test]
+    fn xlsx_cells_are_decimal_strings(values in proptest::collection::vec(
+        (-1_000_000_000_000i64..1_000_000_000_000i64, 0u32..=6u32), 1..12)) {
+        let cells: Vec<String> = values
+            .iter()
+            .map(|(mantissa, scale)| Decimal::new(*mantissa, *scale).to_string())
+            .collect();
+        let text = xlsx_text(&xlsx_of(&cells)).unwrap();
+        let got: Vec<&str> = text.split('\t').collect();
+        prop_assert_eq!(got, cells.iter().map(String::as_str).collect::<Vec<_>>());
+    }
+}
+
+/// INVARIANT 4 and D-32: a file the reader can't read is an error that
+/// says so, not a panic out of the server.
+#[test]
+fn a_pdf_that_crashes_the_reader_is_failed_not_fatal() {
+    for bytes in [
+        &b"not a pdf at all"[..],
+        &b"%PDF-1.7\n1 0 obj << /Type /Catalog"[..],
+        &b""[..],
+    ] {
+        assert!(
+            matches!(pdf_text(bytes), Err(ExtractError::Pdf(_))),
+            "{bytes:?}"
+        );
+    }
+    // The input cap is a refusal too, before the reader runs.
+    let too_big = vec![0u8; MAX_BYTES + 1];
+    assert!(matches!(
+        pdf_text(&too_big),
+        Err(ExtractError::TooLarge { .. })
+    ));
+    assert_eq!(text_of(&too_big, "application/pdf", "x.pdf"), None);
+    // No page past the last, and no zero-sized page.
+    let pdf = a_pdf();
+    assert!(matches!(
+        page_png(&pdf, 30, 600),
+        Err(ExtractError::PageOutOfRange { .. })
+    ));
+    assert!(matches!(page_png(&pdf, 0, 0), Err(ExtractError::NoPixels)));
+}

crates/ledger-import/tests/spec_import.rs

@@ -3347,64 +3347,10 @@ fn pdf_keeps_chinese_japanese_and_korean_names() {
 
 /// The text a PDF draws, each line from the top of each page down, its
 /// glyphs left to right, each the text its font's ToUnicode map gives
-/// back, read with hayro (the viewer's own PDF reader): what a reader
-/// copies, searches and speaks.
+/// back: the one extractor (`ledger_import::extract`, plan P6), which the
+/// server and the viewer read receipts through too.
 fn pdf_lines(pdf: &[u8]) -> Vec<Vec<String>> {
-    use hayro_interpret::font::GlyphRun;
-    use hayro_interpret::hayro_cmap::BfString;
-    use hayro_interpret::hayro_syntax::Pdf;
-    use hayro_interpret::{
-        BlendMode, ClipPath, Context, Device, DrawMode, DrawProps, Image, ImageDrawProps,
-        InterpreterCache, InterpreterSettings, SoftMask, interpret_page,
-    };
-    use kurbo::{Affine, BezPath, Point, Rect};
-    /// Each glyph drawn: its baseline (tenths of a point), x, and text.
-    struct Glyphs(Vec<(i64, f64, String)>);
-    impl<'a> Device<'a> for Glyphs {
-        fn draw_path(&mut self, _: &BezPath, _: DrawProps<'a>, _: &DrawMode) {}
-        fn push_clip_path(&mut self, _: &ClipPath) {}
-        fn push_transparency_group(&mut self, _: f32, _: Option<SoftMask<'a>>, _: BlendMode) {}
-        fn draw_glyph_run(&mut self, run: &GlyphRun<'_, 'a>, props: DrawProps<'a>, _: &DrawMode) {
-            for g in run.glyphs() {
-                let at = (props.transform * g.transform()) * Point::ZERO;
-                let text = match g.as_unicode() {
-                    Some(BfString::Char(c)) => c.to_string(),
-                    Some(BfString::String(s)) => s,
-                    // A mark or a ligature's second glyph: its text is
-                    // with the cluster's first.
-                    None => String::new(),
-                };
-                self.0.push(((at.y * 10.0).round() as i64, at.x, text));
-            }
-        }
-        fn draw_image(&mut self, _: Image<'a, '_>, _: ImageDrawProps<'a>) {}
-        fn pop_clip(&mut self) {}
-        fn pop_transparency_group(&mut self) {}
-    }
-    let pdf = Pdf::new(pdf.to_vec()).unwrap();
-    let cache = InterpreterCache::new();
-    let mut out = Vec::new();
-    for page in pdf.pages().iter() {
-        let mut ctx = Context::new(
-            Affine::IDENTITY,
-            Rect::new(0.0, 0.0, 1.0, 1.0),
-            &cache,
-            pdf.xref(),
-            InterpreterSettings::default(),
-        );
-        let mut glyphs = Glyphs(Vec::new());
-        interpret_page(page, &mut ctx, &mut glyphs);
-        // PDF space runs up the page: the highest baseline first.
-        let mut lines: BTreeMap<std::cmp::Reverse<i64>, Vec<(f64, String)>> = BTreeMap::new();
-        for (y, x, t) in glyphs.0 {
-            lines.entry(std::cmp::Reverse(y)).or_default().push((x, t));
-        }
-        for (_, mut line) in lines {
-            line.sort_by(|a, b| a.0.total_cmp(&b.0));
-            out.push(line.into_iter().map(|(_, t)| t).collect());
-        }
-    }
-    out
+    ledger_import::extract::pdf_lines(pdf).unwrap()
 }
 
 /// statement_pdf INVARIANT 4 (P3 review round 2): a company and accounts