wip: pause checkpoint — P6 feature complete, gate interrupted (not for merge)
This commit is contained in:
swaits committed
1 parent
cc4f104f06
commit
ce36640874
68 files changed
+4592
-374
No files matched your search
@@ -0,0 +1,126 @@
|
||||
//! The one extractor (plan P6 step 1): a PDF's text and pages, and an
|
||||
//! XLSX's cells as exact decimal strings.
|
||||
//!
|
||||
//! Expected behavior comes from `crates/ledger-import/src/extract.rs`'s
|
||||
//! INVARIANTS, which come from plan/served.md P6 step 1 and D-15, D-32 and
|
||||
//! D-53.
|
||||
|
||||
use std::io::{Cursor, Write};
|
||||
|
||||
use ledger_import::extract::{ExtractError, MAX_BYTES, page_png, pdf_text, text_of, xlsx_text};
|
||||
use ledger_import::report::{ReportMeta, Row, RowKind, statement};
|
||||
use ledger_import::statement_pdf;
|
||||
use proptest::prelude::*;
|
||||
use rust_decimal::Decimal;
|
||||
|
||||
fn a_pdf() -> Vec<u8> {
|
||||
let st = statement(
|
||||
"Receipt",
|
||||
"March 2026".to_string(),
|
||||
&["Total"],
|
||||
vec![Row {
|
||||
kind: RowKind::Line,
|
||||
label: "Rent".into(),
|
||||
amounts: vec![Some(Decimal::new(10000, 2))],
|
||||
currency: true,
|
||||
rule_above: false,
|
||||
}],
|
||||
);
|
||||
let meta = ReportMeta {
|
||||
company: "Fernhill Labs LLC".into(),
|
||||
basis: "Accrual".into(),
|
||||
prepared_on: chrono::NaiveDate::from_ymd_opt(2026, 3, 2).unwrap(),
|
||||
journal_seq: 1,
|
||||
};
|
||||
statement_pdf(&st, &meta).unwrap()
|
||||
}
|
||||
|
||||
/// An XLSX holding one sheet whose cells are `values` (numbers as written,
|
||||
/// not through a float).
|
||||
fn xlsx_of(values: &[String]) -> Vec<u8> {
|
||||
let mut writer = zip::ZipWriter::new(Cursor::new(Vec::new()));
|
||||
let options = zip::write::SimpleFileOptions::default();
|
||||
writer
|
||||
.start_file("xl/worksheets/sheet1.xml", options)
|
||||
.unwrap();
|
||||
let cells: String = values
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, v)| format!(r#"<c r="A{}"><v>{v}</v></c>"#, i + 1))
|
||||
.collect();
|
||||
writer
|
||||
.write_all(
|
||||
format!(
|
||||
r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData><row r="1">{cells}</row></sheetData></worksheet>"#
|
||||
)
|
||||
.as_bytes(),
|
||||
)
|
||||
.unwrap();
|
||||
writer.finish().unwrap().into_inner()
|
||||
}
|
||||
|
||||
/// INVARIANT 1: the same bytes give the same text and the same PNG bytes,
|
||||
/// every run.
|
||||
#[test]
|
||||
fn extraction_deterministic() {
|
||||
let pdf = a_pdf();
|
||||
let first = pdf_text(&pdf).unwrap();
|
||||
assert_eq!(first, pdf_text(&pdf).unwrap(), "the text");
|
||||
assert!(first.contains("Rent"), "{first}");
|
||||
let first_png = page_png(&pdf, 0, 600).unwrap();
|
||||
assert_eq!(first_png, page_png(&pdf, 0, 600).unwrap(), "the page");
|
||||
assert!(first_png.starts_with(b"\x89PNG\r\n\x1a\n"), "a PNG");
|
||||
// A plain text file reads back exactly; an image has no text (D-15).
|
||||
assert_eq!(
|
||||
text_of(b"date,amount\n2026-03-01,12.50\n", "text/csv", "bank.csv").as_deref(),
|
||||
Some("date,amount\n2026-03-01,12.50\n")
|
||||
);
|
||||
assert_eq!(text_of(b"\x89PNG\r\n\x1a\n", "image/png", "r.png"), None);
|
||||
}
|
||||
|
||||
// INVARIANT 2: an XLSX cell's number is the exact decimal string the
|
||||
// sheet XML holds, never a binary float.
|
||||
proptest! {
|
||||
#[test]
|
||||
fn xlsx_cells_are_decimal_strings(values in proptest::collection::vec(
|
||||
(-1_000_000_000_000i64..1_000_000_000_000i64, 0u32..=6u32), 1..12)) {
|
||||
let cells: Vec<String> = values
|
||||
.iter()
|
||||
.map(|(mantissa, scale)| Decimal::new(*mantissa, *scale).to_string())
|
||||
.collect();
|
||||
let text = xlsx_text(&xlsx_of(&cells)).unwrap();
|
||||
let got: Vec<&str> = text.split('\t').collect();
|
||||
prop_assert_eq!(got, cells.iter().map(String::as_str).collect::<Vec<_>>());
|
||||
}
|
||||
}
|
||||
|
||||
/// INVARIANT 4 and D-32: a file the reader can't read is an error that
|
||||
/// says so, not a panic out of the server.
|
||||
#[test]
|
||||
fn a_pdf_that_crashes_the_reader_is_failed_not_fatal() {
|
||||
for bytes in [
|
||||
&b"not a pdf at all"[..],
|
||||
&b"%PDF-1.7\n1 0 obj << /Type /Catalog"[..],
|
||||
&b""[..],
|
||||
] {
|
||||
assert!(
|
||||
matches!(pdf_text(bytes), Err(ExtractError::Pdf(_))),
|
||||
"{bytes:?}"
|
||||
);
|
||||
}
|
||||
// The input cap is a refusal too, before the reader runs.
|
||||
let too_big = vec![0u8; MAX_BYTES + 1];
|
||||
assert!(matches!(
|
||||
pdf_text(&too_big),
|
||||
Err(ExtractError::TooLarge { .. })
|
||||
));
|
||||
assert_eq!(text_of(&too_big, "application/pdf", "x.pdf"), None);
|
||||
// No page past the last, and no zero-sized page.
|
||||
let pdf = a_pdf();
|
||||
assert!(matches!(
|
||||
page_png(&pdf, 30, 600),
|
||||
Err(ExtractError::PageOutOfRange { .. })
|
||||
));
|
||||
assert!(matches!(page_png(&pdf, 0, 0), Err(ExtractError::NoPixels)));
|
||||
}
|
||||
@@ -3347,64 +3347,10 @@ fn pdf_keeps_chinese_japanese_and_korean_names() {
|
||||
|
||||
/// The text a PDF draws, each line from the top of each page down, its
|
||||
/// glyphs left to right, each the text its font's ToUnicode map gives
|
||||
/// back, read with hayro (the viewer's own PDF reader): what a reader
|
||||
/// copies, searches and speaks.
|
||||
/// back: the one extractor (`ledger_import::extract`, plan P6), which the
|
||||
/// server and the viewer read receipts through too.
|
||||
fn pdf_lines(pdf: &[u8]) -> Vec<Vec<String>> {
|
||||
use hayro_interpret::font::GlyphRun;
|
||||
use hayro_interpret::hayro_cmap::BfString;
|
||||
use hayro_interpret::hayro_syntax::Pdf;
|
||||
use hayro_interpret::{
|
||||
BlendMode, ClipPath, Context, Device, DrawMode, DrawProps, Image, ImageDrawProps,
|
||||
InterpreterCache, InterpreterSettings, SoftMask, interpret_page,
|
||||
};
|
||||
use kurbo::{Affine, BezPath, Point, Rect};
|
||||
/// Each glyph drawn: its baseline (tenths of a point), x, and text.
|
||||
struct Glyphs(Vec<(i64, f64, String)>);
|
||||
impl<'a> Device<'a> for Glyphs {
|
||||
fn draw_path(&mut self, _: &BezPath, _: DrawProps<'a>, _: &DrawMode) {}
|
||||
fn push_clip_path(&mut self, _: &ClipPath) {}
|
||||
fn push_transparency_group(&mut self, _: f32, _: Option<SoftMask<'a>>, _: BlendMode) {}
|
||||
fn draw_glyph_run(&mut self, run: &GlyphRun<'_, 'a>, props: DrawProps<'a>, _: &DrawMode) {
|
||||
for g in run.glyphs() {
|
||||
let at = (props.transform * g.transform()) * Point::ZERO;
|
||||
let text = match g.as_unicode() {
|
||||
Some(BfString::Char(c)) => c.to_string(),
|
||||
Some(BfString::String(s)) => s,
|
||||
// A mark or a ligature's second glyph: its text is
|
||||
// with the cluster's first.
|
||||
None => String::new(),
|
||||
};
|
||||
self.0.push(((at.y * 10.0).round() as i64, at.x, text));
|
||||
}
|
||||
}
|
||||
fn draw_image(&mut self, _: Image<'a, '_>, _: ImageDrawProps<'a>) {}
|
||||
fn pop_clip(&mut self) {}
|
||||
fn pop_transparency_group(&mut self) {}
|
||||
}
|
||||
let pdf = Pdf::new(pdf.to_vec()).unwrap();
|
||||
let cache = InterpreterCache::new();
|
||||
let mut out = Vec::new();
|
||||
for page in pdf.pages().iter() {
|
||||
let mut ctx = Context::new(
|
||||
Affine::IDENTITY,
|
||||
Rect::new(0.0, 0.0, 1.0, 1.0),
|
||||
&cache,
|
||||
pdf.xref(),
|
||||
InterpreterSettings::default(),
|
||||
);
|
||||
let mut glyphs = Glyphs(Vec::new());
|
||||
interpret_page(page, &mut ctx, &mut glyphs);
|
||||
// PDF space runs up the page: the highest baseline first.
|
||||
let mut lines: BTreeMap<std::cmp::Reverse<i64>, Vec<(f64, String)>> = BTreeMap::new();
|
||||
for (y, x, t) in glyphs.0 {
|
||||
lines.entry(std::cmp::Reverse(y)).or_default().push((x, t));
|
||||
}
|
||||
for (_, mut line) in lines {
|
||||
line.sort_by(|a, b| a.0.total_cmp(&b.0));
|
||||
out.push(line.into_iter().map(|(_, t)| t).collect());
|
||||
}
|
||||
}
|
||||
out
|
||||
ledger_import::extract::pdf_lines(pdf).unwrap()
|
||||
}
|
||||
|
||||
/// statement_pdf INVARIANT 4 (P3 review round 2): a company and accounts
|
||||
|
||||
Reference in new issue
Block a user