merge: P6, the evidence pipeline (branch P6, checkpoint ce366408)
The last of the paused branches, from its "not for merge" checkpoint:
extraction in ledger-import (text, pages, page images; capped, caught),
journaled as attachment.text_extracted; dedup by hash with
books.find_attachment and books.consolidate_duplicates; annotations and
books.set_entry_settles; the company record categories; the two fuzz
targets; feature attachment_text. Nineteen files conflicted with the
four P4b merges, resolved as plan 8.0.2 says: P6 retakes schema 18 (it
had taken 16, which S4.3 holds; S5 holds 17); the tool table is 129 and
the destructive count 31; both sides' appends kept with their seams
restored; one B-27 paragraph naming every feature; in the old-books
fixture P6's hand-written reimbursement is entry 7 (its ids collided
with S4.3's invoice); the replay fuzzer's list gains S7's
invoice.written_off; S4.3's issued-PDF test expects the extraction
record P6 journals right after a filing.
One integration defect between S7 and P6, found by the arap property
test and fixed here: under P6's dedup, a write-off whose evidence bytes
were filed already (the same collection letter for a second write-off of
the same invoice) was refused only after its entry had been journaled,
at the final link to the invoice ("already linked"). The write-off now
resolves the dedup before anything is journaled and keeps a link the
record already has to the invoice instead of refusing it; an explicit
existing_attachment_id already on an entry is still refused (S7's
one-receipt-once rule). P6's gate never ran before the pause; it runs on
main after this merge, and its ledger-store -> ledger-import dependency
is the open review point (plan 8.0.1).
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HS8WZziDnyxunC23MUxrLp
This commit is contained in:
swaitsandClaude Fable 5.1 committed
commit
2ad192bbcd
70 files changed
+4618
-384
No files matched your search
@@ -8,6 +8,11 @@ description = "Durable persistence for wanderbooks: a hash-chained JSONL journal
|
||||
|
||||
[dependencies]
|
||||
ledger-core = { path = "../ledger-core" }
|
||||
# The one extractor (plan P6 step 1, D-53): the writer journals a file's
|
||||
# text at attach time and on `wanderbooks books extract`, so the extractor
|
||||
# must be callable here rather than only from the server. It is pure and
|
||||
# deterministic (no I/O, no network, no time source).
|
||||
ledger-import = { path = "../ledger-import" }
|
||||
rusqlite = { version = "0.32", features = ["bundled"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
|
||||
@@ -63,6 +63,7 @@ pub const KNOWN_FEATURES: &[&str] = &[
|
||||
DOCUMENT_LINE_PROJECTS,
|
||||
DOCUMENT_STANDING,
|
||||
DOCUMENTS_V2,
|
||||
ATTACHMENT_TEXT,
|
||||
];
|
||||
|
||||
/// Feature: `period.closed` closes to members with `"rounding":
|
||||
@@ -198,6 +199,15 @@ pub const DOCUMENT_STANDING: &str = "document_standing";
|
||||
/// discount and the fee, or refuse the unknown values, silently.
|
||||
pub const DOCUMENTS_V2: &str = "documents_v2";
|
||||
|
||||
/// Feature: the evidence pipeline (plan P6): `attachment.text_extracted`
|
||||
/// (a file's text, journaled as an extraction record), `attachment.annotated`
|
||||
/// (the facts a human read off a file), `entry.settles` (which entries a
|
||||
/// payment pays back), and `attachment.withdrawn` carrying `duplicate_of`.
|
||||
/// An older version doesn't know those event types, or would withdraw a
|
||||
/// duplicate without saying what it duplicates; it stops at the
|
||||
/// declaration instead (rules.md B-27, B-44).
|
||||
pub const ATTACHMENT_TEXT: &str = "attachment_text";
|
||||
|
||||
/// The line type that declares the journal format.
|
||||
pub const FORMAT_SET: &str = "journal.format_set";
|
||||
|
||||
@@ -218,7 +228,9 @@ pub const EVENT_TYPES: &[&str] = &[
|
||||
"asset.serial_set",
|
||||
"asset.tagged",
|
||||
"asset.tax_basis_set",
|
||||
"attachment.annotated",
|
||||
"attachment.linked",
|
||||
"attachment.text_extracted",
|
||||
"attachment.unlinked",
|
||||
"attachment.withdrawn",
|
||||
"bill.created",
|
||||
@@ -239,6 +251,7 @@ pub const EVENT_TYPES: &[&str] = &[
|
||||
"entry.party_set",
|
||||
"entry.posted",
|
||||
"entry.reversed",
|
||||
"entry.settles",
|
||||
"entry.vendor_set",
|
||||
"fx.rate_set",
|
||||
"invoice.created",
|
||||
@@ -860,6 +873,40 @@ mod tests {
|
||||
assert_eq!(up.adds(), vec!["the journal feature document_standing"]);
|
||||
}
|
||||
|
||||
/// The version before P6 (every feature through `documents_v2`) stops
|
||||
/// at a declaration of `attachment_text` as newer, never dropping a
|
||||
/// file's extraction, annotations or settlements; and books that
|
||||
/// declared every earlier feature still wait for it (B-27, D-39).
|
||||
#[test]
|
||||
fn attachment_text_is_its_own_feature() {
|
||||
assert!(KNOWN_FEATURES.contains(&ATTACHMENT_TEXT));
|
||||
let before: Vec<&str> = KNOWN_FEATURES
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|f| *f != ATTACHMENT_TEXT)
|
||||
.collect();
|
||||
let l = line(
|
||||
FORMAT_SET,
|
||||
json!({"format": 2, "features": [DOCUMENTS_V2, ATTACHMENT_TEXT]}),
|
||||
);
|
||||
match declared_knowing(&l, &before) {
|
||||
Err(StoreError::NewerJournal { need, .. }) => {
|
||||
assert_eq!(need, "journal features attachment_text")
|
||||
}
|
||||
other => panic!("expected NewerJournal, got {other:?}"),
|
||||
}
|
||||
let older = JournalFormat {
|
||||
format: 2,
|
||||
features: before.iter().map(|f| f.to_string()).collect(),
|
||||
};
|
||||
assert!(matches!(
|
||||
older.require_feature(ATTACHMENT_TEXT),
|
||||
Err(StoreError::FormatUpgradePending { .. })
|
||||
));
|
||||
let up = older.upgrade().expect("waits for it");
|
||||
assert_eq!(up.adds(), vec!["the journal feature attachment_text"]);
|
||||
}
|
||||
|
||||
/// The version before P4b S4.2 (every feature through
|
||||
/// `document_standing`) stops at a declaration of `documents_v2` as
|
||||
/// newer, never restoring a party's email or default terms as absent
|
||||
|
||||
@@ -47,8 +47,8 @@ pub use format::{FormatUpgrade, JournalFormat, SUPPORTED_JOURNAL_FORMAT};
|
||||
pub use journal::{GENESIS_PREV, JournalLine, JournalWriter};
|
||||
pub use snapshot::{DiskCheck, Snapshot};
|
||||
pub use store::{
|
||||
Attachment, BillPreview, DepreciationRun, DisposalPreview, EntryRow, EventRow, InvoicePreview,
|
||||
JOURNAL_FILE, LEGACY_DB_FILE, LOCK_FILE, MAX_RECURRING_RUN, PayrollLineInput, PreviewPosting,
|
||||
Reconciliation, Statement, Store, Subject, VerifyReport, WriteOffEvidence, database_in_use,
|
||||
db_file, db_file_for, is_database_file,
|
||||
Attachment, BillPreview, DepreciationRun, DisposalPreview, EntryRow, EventRow, Extraction,
|
||||
ExtractionStatus, InvoicePreview, JOURNAL_FILE, LEGACY_DB_FILE, LOCK_FILE, MAX_RECURRING_RUN,
|
||||
PayrollLineInput, PreviewPosting, Reconciliation, Statement, Store, Subject, VerifyReport,
|
||||
WriteOffEvidence, database_in_use, db_file, db_file_for, is_database_file,
|
||||
};
|
||||
@@ -19,7 +19,9 @@ use chrono::NaiveDate;
|
||||
use ledger_core::arap::{SendKind, Sent, SentVia};
|
||||
use ledger_core::conventions::{Approval, ConventionKind, NewConvention, NewDecision, NewOpenItem};
|
||||
use ledger_core::settings::{Change, Source};
|
||||
use ledger_core::{ClassId, ProjectFacts, ProjectKind, ProjectStatus};
|
||||
use ledger_core::{
|
||||
Annotation, AnnotationFields, ClassId, ProjectFacts, ProjectKind, ProjectStatus,
|
||||
};
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
/// The journal format these events need: an older reader stops at the
|
||||
@@ -466,6 +468,162 @@ pub fn invoice_written_off_of(p: &Value) -> Result<WrittenOff, StoreError> {
|
||||
})
|
||||
}
|
||||
|
||||
/// `attachment.text_extracted {attachment_id, text_sha256, extractor,
|
||||
/// version, pages, status}` (plan P6, feature `attachment_text`).
|
||||
pub fn text_extracted(
|
||||
attachment_id: u64,
|
||||
text_sha256: Option<&str>,
|
||||
extractor: &str,
|
||||
version: &str,
|
||||
pages: u32,
|
||||
status: &str,
|
||||
) -> Value {
|
||||
json!({
|
||||
"attachment_id": attachment_id,
|
||||
"text_sha256": text_sha256,
|
||||
"extractor": extractor,
|
||||
"version": version,
|
||||
"pages": pages,
|
||||
"status": status,
|
||||
})
|
||||
}
|
||||
|
||||
/// Read an `attachment.text_extracted`: `(attachment_id, text_sha256,
|
||||
/// extractor, version, pages, status)`. A status this version doesn't know
|
||||
/// is newer (B-27: never "corrupt" for newer).
|
||||
pub fn text_extracted_of(line: &crate::journal::JournalLine) -> Result<TextExtracted, StoreError> {
|
||||
const E: &str = "attachment.text_extracted";
|
||||
let p = &line.payload;
|
||||
let status = str_of(p, E, "status")?;
|
||||
if !matches!(status.as_str(), "ok" | "none" | "failed") {
|
||||
return Err(crate::format::newer_content(
|
||||
line,
|
||||
format!("the extraction status {status:?}"),
|
||||
));
|
||||
}
|
||||
Ok((
|
||||
u64_of(p, E, "attachment_id")?,
|
||||
opt_str(p, E, "text_sha256")?,
|
||||
str_of(p, E, "extractor")?,
|
||||
str_of(p, E, "version")?,
|
||||
u32::try_from(u64_of(p, E, "pages")?).map_err(|_| corrupt(E, "pages"))?,
|
||||
status,
|
||||
))
|
||||
}
|
||||
|
||||
/// What an `attachment.text_extracted` says: attachment id, text blob,
|
||||
/// extractor, version, pages, status.
|
||||
pub type TextExtracted = (u64, Option<String>, String, String, u32, String);
|
||||
|
||||
/// The annotation fields this version knows (the closed list, plan P6
|
||||
/// step 5).
|
||||
const ANNOTATION_FIELDS: &[&str] = &[
|
||||
"receipt_number",
|
||||
"payee",
|
||||
"amount",
|
||||
"date",
|
||||
"instrument",
|
||||
"category",
|
||||
"source",
|
||||
"reimbursable_to",
|
||||
];
|
||||
|
||||
/// `attachment.annotated {attachment_id, fields{...}, note?}`: only the
|
||||
/// fields given are written, so a canonical payload.
|
||||
pub fn annotated(a: &Annotation) -> Value {
|
||||
let mut fields = Map::new();
|
||||
put(&mut fields, "receipt_number", a.fields.receipt_number);
|
||||
put(&mut fields, "payee", a.fields.payee.clone());
|
||||
put(
|
||||
&mut fields,
|
||||
"amount",
|
||||
a.fields.amount.map(|v| v.to_string()),
|
||||
);
|
||||
put(&mut fields, "date", a.fields.date.map(|d| d.to_string()));
|
||||
put(&mut fields, "instrument", a.fields.instrument.clone());
|
||||
put(&mut fields, "category", a.fields.category.clone());
|
||||
put(&mut fields, "source", a.fields.source.clone());
|
||||
put(
|
||||
&mut fields,
|
||||
"reimbursable_to",
|
||||
a.fields.reimbursable_to.clone(),
|
||||
);
|
||||
let mut o = Map::new();
|
||||
o.insert("attachment_id".into(), json!(a.attachment_id));
|
||||
o.insert("fields".into(), Value::Object(fields));
|
||||
put(&mut o, "note", a.note.clone());
|
||||
Value::Object(o)
|
||||
}
|
||||
|
||||
/// Read an `attachment.annotated`. A field this version doesn't know, or a
|
||||
/// mistyped one, is judged by its kind: unknown is newer (a later version
|
||||
/// wrote it, B-27), mistyped is corrupt.
|
||||
pub fn annotated_of(line: &crate::journal::JournalLine) -> Result<Annotation, StoreError> {
|
||||
const E: &str = "attachment.annotated";
|
||||
let p = &line.payload;
|
||||
let fields = p
|
||||
.get("fields")
|
||||
.filter(|v| v.is_object())
|
||||
.ok_or_else(|| corrupt(E, "fields"))?;
|
||||
for key in fields.as_object().expect("object").keys() {
|
||||
if !ANNOTATION_FIELDS.contains(&key.as_str()) {
|
||||
return Err(crate::format::newer_content(
|
||||
line,
|
||||
format!("the annotation field {key:?}"),
|
||||
));
|
||||
}
|
||||
}
|
||||
let text = |k: &str| opt_str(fields, E, k);
|
||||
let receipt_number = match fields.get("receipt_number") {
|
||||
None | Some(Value::Null) => None,
|
||||
Some(v) => Some(
|
||||
v.as_u64()
|
||||
.and_then(|n| u32::try_from(n).ok())
|
||||
.ok_or_else(|| corrupt(E, "receipt_number"))?,
|
||||
),
|
||||
};
|
||||
let amount = match text("amount")? {
|
||||
None => None,
|
||||
Some(s) => Some(
|
||||
s.parse()
|
||||
.map_err(|_| StoreError::JournalCorrupt(format!("{E}: amount")))?,
|
||||
),
|
||||
};
|
||||
let date = opt_date(fields, E, "date")?;
|
||||
Ok(Annotation {
|
||||
attachment_id: u64_of(p, E, "attachment_id")?,
|
||||
fields: AnnotationFields {
|
||||
receipt_number,
|
||||
payee: text("payee")?,
|
||||
amount,
|
||||
date,
|
||||
instrument: text("instrument")?,
|
||||
category: text("category")?,
|
||||
source: text("source")?,
|
||||
reimbursable_to: text("reimbursable_to")?,
|
||||
},
|
||||
note: opt_str(p, E, "note")?,
|
||||
})
|
||||
}
|
||||
|
||||
/// `entry.settles {entry_id, settles: [entry_id]}` (plan P6).
|
||||
pub fn entry_settles(entry: u64, settles: &[u64]) -> Value {
|
||||
json!({ "entry_id": entry, "settles": settles })
|
||||
}
|
||||
|
||||
/// Read an `entry.settles`: `(entry, settles)`.
|
||||
pub fn entry_settles_of(p: &Value) -> Result<(u64, Vec<u64>), StoreError> {
|
||||
const E: &str = "entry.settles";
|
||||
let settles = p
|
||||
.get("settles")
|
||||
.and_then(Value::as_array)
|
||||
.ok_or_else(|| corrupt(E, "settles"))?
|
||||
.iter()
|
||||
.map(|v| v.as_u64().ok_or_else(|| corrupt(E, "settles")))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
Ok((u64_of(p, E, "entry_id")?, settles))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -549,6 +707,50 @@ mod tests {
|
||||
entry_approved_of(&entry_approved(5, &a), 9).unwrap(),
|
||||
(5, a)
|
||||
);
|
||||
let line = crate::journal::JournalLine {
|
||||
seq: 11,
|
||||
ts: "2026-10-27T00:00:00Z".into(),
|
||||
line_type: "attachment.annotated".into(),
|
||||
payload: annotated(&Annotation {
|
||||
attachment_id: 3,
|
||||
fields: AnnotationFields {
|
||||
receipt_number: Some(7),
|
||||
payee: Some("Harbor Design".into()),
|
||||
amount: Some("12.34".parse().unwrap()),
|
||||
date: NaiveDate::from_ymd_opt(2026, 3, 1),
|
||||
instrument: Some("Check 1234".into()),
|
||||
category: None,
|
||||
source: Some("a folder".into()),
|
||||
reimbursable_to: Some("Avery Quill".into()),
|
||||
},
|
||||
note: Some("read on 2026-03-02".into()),
|
||||
}),
|
||||
prev: String::new(),
|
||||
};
|
||||
let read = annotated_of(&line).unwrap();
|
||||
assert_eq!(annotated(&read), line.payload);
|
||||
assert_eq!(
|
||||
entry_settles_of(&entry_settles(4, &[1, 2])).unwrap(),
|
||||
(4, vec![1, 2])
|
||||
);
|
||||
let x = crate::journal::JournalLine {
|
||||
seq: 12,
|
||||
ts: "2026-10-27T00:00:00Z".into(),
|
||||
line_type: "attachment.text_extracted".into(),
|
||||
payload: text_extracted(3, Some("00ff"), "wanderbooks", "1", 2, "ok"),
|
||||
prev: String::new(),
|
||||
};
|
||||
assert_eq!(
|
||||
text_extracted_of(&x).unwrap(),
|
||||
(
|
||||
3,
|
||||
Some("00ff".into()),
|
||||
"wanderbooks".into(),
|
||||
"1".into(),
|
||||
2,
|
||||
"ok".into()
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
/// INVARIANT 1.
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
//! is the current version.
|
||||
|
||||
/// Current schema version.
|
||||
pub const SCHEMA_VERSION: u32 = 17;
|
||||
pub const SCHEMA_VERSION: u32 = 18;
|
||||
|
||||
/// Migrations, in order. `MIGRATIONS[i]` migrates from version `i` to `i+1`.
|
||||
pub const MIGRATIONS: &[&str] = &[
|
||||
@@ -434,5 +434,39 @@ CREATE TABLE IF NOT EXISTS invoice_sends(
|
||||
reference TEXT NULL,
|
||||
note TEXT NULL
|
||||
);
|
||||
"#,
|
||||
r#"
|
||||
-- v18: evidence (plan P6, feature attachment_text): what was read out of
|
||||
-- a file (`attachment.text_extracted`), the facts a human annotated it
|
||||
-- with (`attachment.annotated`), and the duplicate an attachment was
|
||||
-- withdrawn as. Extraction is its own record per (attachment, extractor,
|
||||
-- version): an extractor upgrade adds one, never replaces one. A text
|
||||
-- hash is the extraction's blob; the text itself is content-addressed in
|
||||
-- the blob store. Amounts stay TEXT (exact decimal), never REAL.
|
||||
CREATE TABLE IF NOT EXISTS extractions(
|
||||
attachment_id INTEGER NOT NULL,
|
||||
extractor TEXT NOT NULL,
|
||||
version TEXT NOT NULL,
|
||||
text_sha256 TEXT NULL,
|
||||
pages INTEGER NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
seq INTEGER NOT NULL,
|
||||
PRIMARY KEY(attachment_id, extractor, version)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_extractions_text ON extractions(text_sha256);
|
||||
CREATE TABLE IF NOT EXISTS annotations(
|
||||
attachment_id INTEGER PRIMARY KEY,
|
||||
receipt_number INTEGER NULL,
|
||||
payee TEXT NULL,
|
||||
amount TEXT NULL,
|
||||
date TEXT NULL,
|
||||
instrument TEXT NULL,
|
||||
category TEXT NULL,
|
||||
source TEXT NULL,
|
||||
reimbursable_to TEXT NULL,
|
||||
note TEXT NULL,
|
||||
seq INTEGER NOT NULL
|
||||
);
|
||||
ALTER TABLE attachments ADD COLUMN duplicate_of INTEGER NULL;
|
||||
"#,
|
||||
];
|
||||
@@ -19,10 +19,10 @@ use crate::error::StoreError;
|
||||
use crate::format::JournalFormat;
|
||||
use crate::journal::{GENESIS_PREV, JournalLine};
|
||||
use crate::store::{
|
||||
Attachment, BLOBS_DIR, JOURNAL_FILE, Reconciliation, Statement, Store, Subject,
|
||||
apply_line_to_sqlite, compare_projections, projection_rows, query_attachment_links,
|
||||
query_attachments, query_reconciliations, query_statements, read_attachment_bytes,
|
||||
replay_line_into_ledger,
|
||||
Attachment, BLOBS_DIR, Extraction, JOURNAL_FILE, Reconciliation, Statement, Store, Subject,
|
||||
apply_line_to_sqlite, compare_projections, projection_rows, query_annotations,
|
||||
query_attachment_links, query_attachments, query_extractions, query_reconciliations,
|
||||
query_statements, read_attachment_bytes, replay_line_into_ledger,
|
||||
};
|
||||
use ledger_core::AccountId;
|
||||
use ledger_core::{Currency, EntryId, Ledger};
|
||||
@@ -261,6 +261,16 @@ impl Snapshot {
|
||||
query_attachments(&self.conn, entry_id)
|
||||
}
|
||||
|
||||
/// Every extraction record, in seq order (as [`Store::extractions`]).
|
||||
pub fn extractions(&self) -> Result<Vec<Extraction>, StoreError> {
|
||||
query_extractions(&self.conn)
|
||||
}
|
||||
|
||||
/// Every annotation, by attachment id (as [`Store::annotations`]).
|
||||
pub fn annotations(&self) -> Result<Vec<ledger_core::Annotation>, StoreError> {
|
||||
query_annotations(&self.conn)
|
||||
}
|
||||
|
||||
/// Every attachment link beyond a receipt's filing entry, as
|
||||
/// (attachment id, subject), by attachment then subject: what
|
||||
/// [`Store::attachment_links`] reads, for every attachment at once.
|
||||
|
||||
@@ -45,6 +45,7 @@ use ledger_core::{
|
||||
payroll::{PayrollAccounts, payroll_entry_plan, payroll_remit_plan},
|
||||
reports,
|
||||
};
|
||||
use ledger_import::extract::{EXTRACTOR as E_EXTRACTOR, VERSION as E_VERSION};
|
||||
use rusqlite::{Connection, OptionalExtension, params};
|
||||
use rust_decimal::Decimal;
|
||||
use serde_json::{Value, json};
|
||||
@@ -299,7 +300,11 @@ pub struct Attachment {
|
||||
pub blob_sha: String,
|
||||
/// Linked entry, if any.
|
||||
pub entry_id: Option<u64>,
|
||||
/// `receipts`, `formation`, `legal`, `tax`, `accounting`, `other`.
|
||||
/// `receipts`, or a document's: `formation`, `legal`, `tax`,
|
||||
/// `accounting`, `statement` (a bank or card statement), `contract`,
|
||||
/// `photo`, `warranty`, `import_source` (a bank export), `notices`,
|
||||
/// `correspondence`, `plans`, `history`, `other` (D-20: every company
|
||||
/// record, not only what the CPA needs).
|
||||
pub category: String,
|
||||
/// Original file name.
|
||||
pub file_name: String,
|
||||
@@ -315,6 +320,64 @@ pub struct Attachment {
|
||||
pub withdrawn_at: Option<String>,
|
||||
/// Why it was withdrawn, if it was.
|
||||
pub withdrawn_reason: Option<String>,
|
||||
/// The attachment it duplicates, when it was withdrawn as one
|
||||
/// (`attachment.withdrawn`'s `duplicate_of`, plan P6 step 4).
|
||||
pub duplicate_of: Option<u64>,
|
||||
}
|
||||
|
||||
/// What reading a file's text found (`attachment.text_extracted`, plan P6
|
||||
/// step 2).
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum ExtractionStatus {
|
||||
/// Text was read.
|
||||
Ok,
|
||||
/// There is no text to read (an image, an empty sheet): D-15's page
|
||||
/// pictures, not OCR.
|
||||
None,
|
||||
/// Reading was tried and failed: a human must read the file.
|
||||
Failed,
|
||||
}
|
||||
|
||||
impl ExtractionStatus {
|
||||
/// Its wire name, as journaled.
|
||||
pub fn key(self) -> &'static str {
|
||||
match self {
|
||||
ExtractionStatus::Ok => "ok",
|
||||
ExtractionStatus::None => "none",
|
||||
ExtractionStatus::Failed => "failed",
|
||||
}
|
||||
}
|
||||
|
||||
/// A wire name back as a status.
|
||||
pub fn parse(key: &str) -> Option<Self> {
|
||||
match key {
|
||||
"ok" => Some(ExtractionStatus::Ok),
|
||||
"none" => Some(ExtractionStatus::None),
|
||||
"failed" => Some(ExtractionStatus::Failed),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One record of reading a file's text, per (attachment, extractor,
|
||||
/// version): an extractor upgrade journals a new record, never replaces
|
||||
/// one.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Extraction {
|
||||
/// The attachment read.
|
||||
pub attachment_id: u64,
|
||||
/// The text's blob (its SHA-256), when there is text.
|
||||
pub text_sha256: Option<String>,
|
||||
/// What read it (`ledger_import::extract::EXTRACTOR`).
|
||||
pub extractor: String,
|
||||
/// That extractor's version.
|
||||
pub version: String,
|
||||
/// How many pages the file has (0 when it has none).
|
||||
pub pages: u32,
|
||||
/// What was found.
|
||||
pub status: ExtractionStatus,
|
||||
/// The journal seq of the record (the latest wins when reading).
|
||||
pub seq: u64,
|
||||
}
|
||||
|
||||
/// Input to [`Store::run_payroll`]: one employee's gross pay and every
|
||||
@@ -3849,7 +3912,13 @@ impl Store {
|
||||
return Err(StoreError::WriteOffNeedsEvidence);
|
||||
};
|
||||
// Every check on the evidence before anything is journaled: a bad
|
||||
// attachment refuses the whole write-off.
|
||||
// attachment refuses the whole write-off. A file whose bytes are
|
||||
// filed already is that record, linked to the write-off's entry and
|
||||
// never a second one (plan P6 step 4, `Self::attach`); it is checked
|
||||
// here as an existing attachment is, except that a link it already
|
||||
// has to this invoice (the same letter evidencing a second write-off
|
||||
// of it) is kept, not refused (found 2026-10-28, merging S7 and P6:
|
||||
// the refusal came after the entry was journaled).
|
||||
if let WriteOffEvidence::Existing(id) = evidence {
|
||||
self.check_linkable(id)?;
|
||||
if self
|
||||
@@ -3862,6 +3931,11 @@ impl Store {
|
||||
));
|
||||
}
|
||||
}
|
||||
if let WriteOffEvidence::File { bytes, .. } = evidence
|
||||
&& let Some(existing) = self.live_attachment_with_sha(&sha256_hex(bytes))?
|
||||
{
|
||||
self.check_linkable(existing.id)?;
|
||||
}
|
||||
// The bytes hit the content-addressed store before anything is
|
||||
// journaled: an orphan blob from a later refusal is harmless
|
||||
// (unreferenced, deduplicated), a journal line naming a blob that
|
||||
@@ -3904,7 +3978,12 @@ impl Store {
|
||||
id
|
||||
}
|
||||
};
|
||||
self.link_attachment(attachment_id, Subject::Invoice(invoice))?;
|
||||
if !self
|
||||
.attachment_links(attachment_id)?
|
||||
.contains(&Subject::Invoice(invoice))
|
||||
{
|
||||
self.link_attachment(attachment_id, Subject::Invoice(invoice))?;
|
||||
}
|
||||
Ok((entry_id, attachment_id))
|
||||
}
|
||||
/// The journal features a settlement needs (rules.md B-27, B-42):
|
||||
@@ -4459,12 +4538,32 @@ impl Store {
|
||||
}
|
||||
|
||||
/// One filing of bytes into the blob store and journal.
|
||||
///
|
||||
/// Plan P6 step 4 (defect 4): bytes already filed (and not withdrawn)
|
||||
/// are one file, not a second record: the existing attachment is
|
||||
/// linked to the new subject and returned, `deduplicated` true. A
|
||||
/// filing with no subject (an unlinked receipt, a document) returns
|
||||
/// the existing record too, for the caller to link later. Legacy
|
||||
/// duplicates stay as they were written.
|
||||
fn attach(&mut self, req: AttachRequest<'_>) -> Result<(u64, String, bool), StoreError> {
|
||||
if let Some(eid) = req.entry_id {
|
||||
self.ledger
|
||||
.get_entry(eid)
|
||||
.ok_or(LedgerError::UnknownEntry(eid))?;
|
||||
}
|
||||
let sha = sha256_hex(req.bytes);
|
||||
if let Some(existing) = self.live_attachment_with_sha(&sha)? {
|
||||
if let Some(e) = req.entry_id {
|
||||
let covered = existing.entry_id == Some(e)
|
||||
|| self
|
||||
.attachment_links(existing.id)?
|
||||
.contains(&Subject::Entry(e));
|
||||
if !covered {
|
||||
self.link_attachment(existing.id, Subject::Entry(e))?;
|
||||
}
|
||||
}
|
||||
return Ok((existing.id, sha, true));
|
||||
}
|
||||
// Bytes hit the content-addressed store first; an orphan blob from a
|
||||
// failed journal append is harmless (unreferenced, deduplicated).
|
||||
let (sha, dedup) = self.blobs.put(req.bytes)?;
|
||||
@@ -4487,9 +4586,212 @@ impl Store {
|
||||
}),
|
||||
)?;
|
||||
apply_line_to_sqlite(&self.conn, &line)?;
|
||||
// Plan P6 step 2: a file is read as it is filed. Books that haven't
|
||||
// declared `attachment_text` yet keep filing (the owner's go-ahead
|
||||
// gates the extraction records, not attaching a receipt); their
|
||||
// old and new files are read by `wanderbooks books extract` after
|
||||
// the upgrade.
|
||||
if self.format.has_feature(crate::format::ATTACHMENT_TEXT) {
|
||||
self.extract_and_journal(id, req.bytes, req.mime_type, req.file_name)?;
|
||||
}
|
||||
Ok((id, sha, dedup))
|
||||
}
|
||||
|
||||
/// The first attachment holding `sha`'s bytes that is not withdrawn.
|
||||
fn live_attachment_with_sha(&self, sha: &str) -> Result<Option<Attachment>, StoreError> {
|
||||
Ok(query_attachments(&self.conn, None)?
|
||||
.into_iter()
|
||||
.find(|a| a.blob_sha.eq_ignore_ascii_case(sha) && a.withdrawn_at.is_none()))
|
||||
}
|
||||
|
||||
/// Read a new attachment's text and journal the record (plan P6 step
|
||||
/// 2): a content-addressed text blob, the extractor and its version,
|
||||
/// the pages, and `ok`, `none` or `failed`. A file with no text still
|
||||
/// gets its `none` record: the books say reading was tried.
|
||||
fn extract_and_journal(
|
||||
&mut self,
|
||||
attachment_id: u64,
|
||||
bytes: &[u8],
|
||||
mime_type: &str,
|
||||
file_name: &str,
|
||||
) -> Result<Extraction, StoreError> {
|
||||
if let Some(already) = self.extraction_for(attachment_id, E_EXTRACTOR, E_VERSION)? {
|
||||
return Ok(already);
|
||||
}
|
||||
let got = ledger_import::extract::extract(bytes, mime_type, file_name);
|
||||
let text_sha = match &got.text {
|
||||
Some(text) => Some(self.blobs.put(text.as_bytes())?.0),
|
||||
None => None,
|
||||
};
|
||||
let line = self.append_journal(
|
||||
"attachment.text_extracted",
|
||||
crate::meta::text_extracted(
|
||||
attachment_id,
|
||||
text_sha.as_deref(),
|
||||
E_EXTRACTOR,
|
||||
E_VERSION,
|
||||
got.pages,
|
||||
got.status.key(),
|
||||
),
|
||||
)?;
|
||||
apply_line_to_sqlite(&self.conn, &line)?;
|
||||
Ok(Extraction {
|
||||
attachment_id,
|
||||
text_sha256: text_sha,
|
||||
extractor: E_EXTRACTOR.into(),
|
||||
version: E_VERSION.into(),
|
||||
pages: got.pages,
|
||||
status: match got.status {
|
||||
ledger_import::extract::ExtractStatus::Ok => ExtractionStatus::Ok,
|
||||
ledger_import::extract::ExtractStatus::None => ExtractionStatus::None,
|
||||
ledger_import::extract::ExtractStatus::Failed => ExtractionStatus::Failed,
|
||||
},
|
||||
seq: line.seq,
|
||||
})
|
||||
}
|
||||
|
||||
/// Read an attachment's text now and journal it (plan P6 step 2;
|
||||
/// idempotent on attachment, extractor and version). A refusal (an
|
||||
/// unknown attachment, books that haven't declared `attachment_text`)
|
||||
/// journals nothing. The backfill
|
||||
/// (`wanderbooks books extract ID`) loops this over old attachments,
|
||||
/// one at a time.
|
||||
pub fn extract_attachment(&mut self, attachment_id: u64) -> Result<Extraction, StoreError> {
|
||||
let a = self
|
||||
.attachment_by_id(attachment_id)?
|
||||
.ok_or(StoreError::UnknownAttachment(attachment_id))?;
|
||||
if let Some(already) = self.extraction_for(attachment_id, E_EXTRACTOR, E_VERSION)? {
|
||||
return Ok(already);
|
||||
}
|
||||
self.ensure_feature(crate::format::ATTACHMENT_TEXT)?;
|
||||
let bytes = self.blobs.get(&a.blob_sha)?;
|
||||
self.extract_and_journal(attachment_id, &bytes, &a.mime_type, &a.file_name)
|
||||
}
|
||||
|
||||
/// The newest extraction record for an attachment, if any.
|
||||
pub fn extraction(&self, attachment_id: u64) -> Result<Option<Extraction>, StoreError> {
|
||||
Ok(query_extractions(&self.conn)?
|
||||
.into_iter()
|
||||
.filter(|x| x.attachment_id == attachment_id)
|
||||
.max_by_key(|x| x.seq))
|
||||
}
|
||||
|
||||
/// The extraction record for exactly this extractor and version.
|
||||
fn extraction_for(
|
||||
&self,
|
||||
attachment_id: u64,
|
||||
extractor: &str,
|
||||
version: &str,
|
||||
) -> Result<Option<Extraction>, StoreError> {
|
||||
Ok(query_extractions(&self.conn)?.into_iter().find(|x| {
|
||||
x.attachment_id == attachment_id && x.extractor == extractor && x.version == version
|
||||
}))
|
||||
}
|
||||
|
||||
/// The text an attachment's newest extraction read, if any.
|
||||
pub fn extraction_text(&self, attachment_id: u64) -> Result<Option<String>, StoreError> {
|
||||
let Some(x) = self.extraction(attachment_id)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(sha) = x.text_sha256 else {
|
||||
return Ok(None);
|
||||
};
|
||||
Ok(Some(
|
||||
String::from_utf8_lossy(&self.blobs.get(&sha)?).into_owned(),
|
||||
))
|
||||
}
|
||||
|
||||
/// Every extraction record, in seq order.
|
||||
pub fn extractions(&self) -> Result<Vec<Extraction>, StoreError> {
|
||||
query_extractions(&self.conn)
|
||||
}
|
||||
|
||||
/// Every annotation, by attachment id (plan P6 step 5).
|
||||
pub fn annotations(&self) -> Result<Vec<ledger_core::Annotation>, StoreError> {
|
||||
query_annotations(&self.conn)
|
||||
}
|
||||
|
||||
/// An attachment by id, withdrawn or not.
|
||||
pub fn attachment_by_id(&self, id: u64) -> Result<Option<Attachment>, StoreError> {
|
||||
Ok(query_attachments(&self.conn, None)?
|
||||
.into_iter()
|
||||
.find(|a| a.id == id))
|
||||
}
|
||||
|
||||
/// Attachments whose bytes hold `text`, case-insensitively: the text
|
||||
/// search of `books.find_attachment` (plan P6 step 2's extraction is
|
||||
/// what makes a receipt searchable by what it says).
|
||||
pub fn attachments_holding_text(&self, text: &str) -> Result<Vec<Attachment>, StoreError> {
|
||||
let needle = text.to_lowercase();
|
||||
let mut out = Vec::new();
|
||||
for a in query_attachments(&self.conn, None)? {
|
||||
if self
|
||||
.extraction_text(a.id)?
|
||||
.is_some_and(|t| t.to_lowercase().contains(&needle))
|
||||
{
|
||||
out.push(a);
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Attachments annotated with `receipt_number`.
|
||||
pub fn attachments_with_receipt_number(
|
||||
&self,
|
||||
receipt_number: u32,
|
||||
) -> Result<Vec<Attachment>, StoreError> {
|
||||
let ids: Vec<i64> = self
|
||||
.conn
|
||||
.prepare("SELECT attachment_id FROM annotations WHERE receipt_number = ?")?
|
||||
.query_map(params![i64::from(receipt_number)], |r| r.get(0))?
|
||||
.collect::<Result<_, _>>()?;
|
||||
Ok(query_attachments(&self.conn, None)?
|
||||
.into_iter()
|
||||
.filter(|a| ids.contains(&(a.id as i64)))
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Record what a human read off a file (`attachment.annotated`, plan
|
||||
/// P6 step 5), replacing an earlier annotation of the same file. The
|
||||
/// fields are the closed list of [`ledger_core::AnnotationFields`];
|
||||
/// a refusal journals nothing.
|
||||
pub fn annotate_attachment(
|
||||
&mut self,
|
||||
annotation: ledger_core::Annotation,
|
||||
) -> Result<ledger_core::Annotation, StoreError> {
|
||||
self.ensure_feature(crate::format::ATTACHMENT_TEXT)?;
|
||||
let a = self
|
||||
.attachment_by_id(annotation.attachment_id)?
|
||||
.ok_or(StoreError::UnknownAttachment(annotation.attachment_id))?;
|
||||
if a.withdrawn_at.is_some() {
|
||||
return Err(StoreError::AttachmentWithdrawn(a.id));
|
||||
}
|
||||
let annotation = annotation.checked()?;
|
||||
self.rehearse(|l| l.annotate_attachment(annotation.clone()))?;
|
||||
let line =
|
||||
self.append_journal("attachment.annotated", crate::meta::annotated(&annotation))?;
|
||||
apply_line_to_sqlite(&self.conn, &line)?;
|
||||
self.ledger.annotate_attachment(annotation.clone())?;
|
||||
Ok(annotation)
|
||||
}
|
||||
|
||||
/// Record which entries an entry settles (`entry.settles`, plan P6
|
||||
/// step 5): what a reimbursement names, and the only source of a
|
||||
/// member-paid entry's "reimbursed" (IZ-18).
|
||||
pub fn set_entry_settles(
|
||||
&mut self,
|
||||
entry: EntryId,
|
||||
settles: &[EntryId],
|
||||
) -> Result<(), StoreError> {
|
||||
self.ensure_feature(crate::format::ATTACHMENT_TEXT)?;
|
||||
self.rehearse(|l| l.set_entry_settles(entry, settles.to_vec()))?;
|
||||
let line =
|
||||
self.append_journal("entry.settles", crate::meta::entry_settles(entry, settles))?;
|
||||
apply_line_to_sqlite(&self.conn, &line)?;
|
||||
self.ledger.set_entry_settles(entry, settles.to_vec())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Register an amortization schedule. Journaled as `schedule.registered`,
|
||||
/// with its `convention` (`daily` or `monthly_even`). A daily schedule
|
||||
/// needs journal feature `schedule_convention` (refused, journaling
|
||||
@@ -5119,6 +5421,32 @@ impl Store {
|
||||
&mut self,
|
||||
attachment_id: u64,
|
||||
reason: &str,
|
||||
) -> Result<(), StoreError> {
|
||||
self.withdraw_attachment_as(attachment_id, reason, None)
|
||||
}
|
||||
|
||||
/// [`Self::withdraw_attachment`] of a duplicate, naming the attachment
|
||||
/// it duplicates (`attachment.withdrawn`'s `duplicate_of`, plan P6
|
||||
/// step 4).
|
||||
pub fn withdraw_attachment_duplicate(
|
||||
&mut self,
|
||||
attachment_id: u64,
|
||||
reason: &str,
|
||||
duplicate_of: u64,
|
||||
) -> Result<(), StoreError> {
|
||||
if duplicate_of == attachment_id {
|
||||
return Err(StoreError::Ledger(LedgerError::InvalidEvidence(
|
||||
"an attachment doesn't duplicate itself".into(),
|
||||
)));
|
||||
}
|
||||
self.withdraw_attachment_as(attachment_id, reason, Some(duplicate_of))
|
||||
}
|
||||
|
||||
fn withdraw_attachment_as(
|
||||
&mut self,
|
||||
attachment_id: u64,
|
||||
reason: &str,
|
||||
duplicate_of: Option<u64>,
|
||||
) -> Result<(), StoreError> {
|
||||
let reason = reason.trim();
|
||||
if reason.is_empty() {
|
||||
@@ -5128,9 +5456,21 @@ impl Store {
|
||||
if let Some(sub) = self.attachment_links(attachment_id)?.first() {
|
||||
return Err(StoreError::AlreadyLinked(attachment_id, sub.describe()));
|
||||
}
|
||||
if let Some(dup) = duplicate_of {
|
||||
// The one it duplicates must be a different attachment, and
|
||||
// `duplicate_of` is a field only books with the feature read
|
||||
// (plan P6).
|
||||
self.ensure_feature(crate::format::ATTACHMENT_TEXT)?;
|
||||
self.attachment_by_id(dup)?
|
||||
.ok_or(StoreError::UnknownAttachment(dup))?;
|
||||
}
|
||||
let line = self.append_journal(
|
||||
"attachment.withdrawn",
|
||||
json!({ "attachment_id": attachment_id, "reason": reason }),
|
||||
json!({
|
||||
"attachment_id": attachment_id,
|
||||
"reason": reason,
|
||||
"duplicate_of": duplicate_of,
|
||||
}),
|
||||
)?;
|
||||
apply_line_to_sqlite(&self.conn, &line)?;
|
||||
Ok(())
|
||||
@@ -5789,6 +6129,20 @@ impl Store {
|
||||
"payroll runs diverged between journal, live ledger, and sqlite projection".into(),
|
||||
));
|
||||
}
|
||||
// The text an extraction recorded is a stored file too: present,
|
||||
// and the hash it names (plan P6 step 2). It is never re-derived
|
||||
// here (an extractor upgrade journals a new record, never replaces
|
||||
// one).
|
||||
for x in query_extractions(&self.conn)? {
|
||||
if let Some(sha) = x.text_sha256 {
|
||||
self.blobs.get(&sha).map_err(|e| {
|
||||
StoreError::VerifyMismatch(format!(
|
||||
"attachment {}'s extracted text doesn't verify: {e}",
|
||||
x.attachment_id
|
||||
))
|
||||
})?;
|
||||
}
|
||||
}
|
||||
// Finally, the whole projection, not just what reports read: every
|
||||
// table, row for row, against a fresh projection of the journal.
|
||||
compare_projections(
|
||||
@@ -5848,7 +6202,9 @@ impl Store {
|
||||
/// The SHA-256 of every stored file the books name (receipts and
|
||||
/// documents, withdrawn ones included).
|
||||
pub fn stored_blob_shas(&self) -> Result<Vec<String>, StoreError> {
|
||||
let mut stmt = self.conn.prepare("SELECT sha256 FROM blobs")?;
|
||||
let mut stmt = self.conn.prepare(
|
||||
"SELECT sha256 FROM blobs UNION SELECT text_sha256 FROM extractions WHERE text_sha256 IS NOT NULL",
|
||||
)?;
|
||||
let shas = stmt
|
||||
.query_map([], |r| r.get::<_, String>(0))?
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
@@ -7031,8 +7387,22 @@ fn replay_line(
|
||||
| "receipt.linked"
|
||||
| "receipt.unlinked"
|
||||
| "document.filed"
|
||||
| "attachment.withdrawn" => {
|
||||
// Not accounting state; the projection records them.
|
||||
| "attachment.withdrawn"
|
||||
| "attachment.text_extracted" => {
|
||||
// Not accounting state; the projection records them. (The text
|
||||
// record is checked for shape here: an unknown status is newer,
|
||||
// rules.md B-27.)
|
||||
if line.line_type == "attachment.text_extracted" {
|
||||
crate::meta::text_extracted_of(line)?;
|
||||
}
|
||||
}
|
||||
"attachment.annotated" => {
|
||||
let a = crate::meta::annotated_of(line)?;
|
||||
ledger.replay_annotate_attachment(a)?;
|
||||
}
|
||||
"entry.settles" => {
|
||||
let (entry, settles) = crate::meta::entry_settles_of(p)?;
|
||||
ledger.replay_set_entry_settles(entry, settles)?;
|
||||
}
|
||||
"payroll.taxes_remitted" => {
|
||||
// Names the remittance entry (rebuilt by the entry events just
|
||||
@@ -8135,14 +8505,56 @@ fn apply_line_rows(conn: &Connection, line: &JournalLine) -> Result<(), StoreErr
|
||||
}
|
||||
"attachment.withdrawn" => {
|
||||
conn.execute(
|
||||
"UPDATE attachments SET withdrawn_at = ?, withdrawn_reason = ? WHERE id = ?",
|
||||
"UPDATE attachments SET withdrawn_at = ?, withdrawn_reason = ?, duplicate_of = ? WHERE id = ?",
|
||||
params![
|
||||
line.ts,
|
||||
get_str(p, "reason")?,
|
||||
p.get("duplicate_of").and_then(|v| v.as_u64()).map(|v| v as i64),
|
||||
get_u64(p, "attachment_id")? as i64,
|
||||
],
|
||||
)?;
|
||||
}
|
||||
"attachment.text_extracted" => {
|
||||
let (id, sha, extractor, version, pages, status) =
|
||||
crate::meta::text_extracted_of(line)?;
|
||||
conn.execute(
|
||||
"INSERT INTO extractions(attachment_id, extractor, version, text_sha256, pages, status, seq)
|
||||
VALUES (?,?,?,?,?,?,?)
|
||||
ON CONFLICT(attachment_id, extractor, version) DO UPDATE SET
|
||||
text_sha256 = excluded.text_sha256, pages = excluded.pages,
|
||||
status = excluded.status, seq = excluded.seq",
|
||||
params![id as i64, extractor, version, sha, i64::from(pages), status, line.seq as i64],
|
||||
)?;
|
||||
}
|
||||
"attachment.annotated" => {
|
||||
let a = crate::meta::annotated_of(line)?;
|
||||
conn.execute(
|
||||
"INSERT INTO annotations(attachment_id, receipt_number, payee, amount, date, instrument, category, source, reimbursable_to, note, seq)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?)
|
||||
ON CONFLICT(attachment_id) DO UPDATE SET
|
||||
receipt_number = excluded.receipt_number, payee = excluded.payee,
|
||||
amount = excluded.amount, date = excluded.date,
|
||||
instrument = excluded.instrument, category = excluded.category,
|
||||
source = excluded.source, reimbursable_to = excluded.reimbursable_to,
|
||||
note = excluded.note, seq = excluded.seq",
|
||||
params![
|
||||
a.attachment_id as i64,
|
||||
a.fields.receipt_number.map(i64::from),
|
||||
a.fields.payee,
|
||||
a.fields.amount.map(|v| v.to_string()),
|
||||
a.fields.date.map(|d| d.to_string()),
|
||||
a.fields.instrument,
|
||||
a.fields.category,
|
||||
a.fields.source,
|
||||
a.fields.reimbursable_to,
|
||||
a.note,
|
||||
line.seq as i64,
|
||||
],
|
||||
)?;
|
||||
}
|
||||
"entry.settles" => {
|
||||
// Rebuilt into the ledger by replay; no report rows.
|
||||
}
|
||||
"asset.registered" => {
|
||||
let cost = &p["cost"];
|
||||
let salvage = &p["salvage_value"];
|
||||
@@ -8920,7 +9332,8 @@ pub(crate) fn query_attachments(
|
||||
let mut sql = String::from(
|
||||
"SELECT a.id, a.blob_sha, a.entry_id, a.category, \
|
||||
COALESCE(a.file_name, b.original_name), COALESCE(a.mime_type, b.mime_type), \
|
||||
b.size_bytes, a.description, a.created_at, a.withdrawn_at, a.withdrawn_reason \
|
||||
b.size_bytes, a.description, a.created_at, a.withdrawn_at, a.withdrawn_reason, \
|
||||
a.duplicate_of \
|
||||
FROM attachments a \
|
||||
JOIN blobs b ON b.sha256 = a.blob_sha",
|
||||
);
|
||||
@@ -8971,9 +9384,61 @@ fn row_to_attachment(r: &rusqlite::Row) -> rusqlite::Result<Attachment> {
|
||||
created_at: r.get(8)?,
|
||||
withdrawn_at: r.get(9)?,
|
||||
withdrawn_reason: r.get(10)?,
|
||||
duplicate_of: r.get::<_, Option<i64>>(11)?.map(|v| v as u64),
|
||||
})
|
||||
}
|
||||
|
||||
/// Every extraction record, in seq order (plan P6).
|
||||
pub(crate) fn query_extractions(conn: &Connection) -> Result<Vec<Extraction>, StoreError> {
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT attachment_id, text_sha256, extractor, version, pages, status, seq \
|
||||
FROM extractions ORDER BY seq, attachment_id, extractor, version",
|
||||
)?;
|
||||
let rows = stmt.query_map([], |r| {
|
||||
let status: String = r.get(5)?;
|
||||
Ok(Extraction {
|
||||
attachment_id: r.get::<_, i64>(0)? as u64,
|
||||
text_sha256: r.get(1)?,
|
||||
extractor: r.get(2)?,
|
||||
version: r.get(3)?,
|
||||
pages: r.get::<_, i64>(4)? as u32,
|
||||
status: ExtractionStatus::parse(&status).unwrap_or(ExtractionStatus::Failed),
|
||||
seq: r.get::<_, i64>(6)? as u64,
|
||||
})
|
||||
})?;
|
||||
Ok(rows.collect::<Result<_, _>>()?)
|
||||
}
|
||||
|
||||
/// Every annotation, by attachment id (plan P6).
|
||||
pub(crate) fn query_annotations(
|
||||
conn: &Connection,
|
||||
) -> Result<Vec<ledger_core::Annotation>, StoreError> {
|
||||
use ledger_core::{Annotation, AnnotationFields};
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT attachment_id, receipt_number, payee, amount, date, instrument, category, \
|
||||
source, reimbursable_to, note FROM annotations ORDER BY attachment_id",
|
||||
)?;
|
||||
let rows = stmt.query_map([], |r| {
|
||||
let amount: Option<String> = r.get(3)?;
|
||||
let date: Option<String> = r.get(4)?;
|
||||
Ok(Annotation {
|
||||
attachment_id: r.get::<_, i64>(0)? as u64,
|
||||
fields: AnnotationFields {
|
||||
receipt_number: r.get::<_, Option<i64>>(1)?.map(|v| v as u32),
|
||||
payee: r.get(2)?,
|
||||
amount: amount.and_then(|a| a.parse().ok()),
|
||||
date: date.and_then(|d| NaiveDate::parse_from_str(&d, "%Y-%m-%d").ok()),
|
||||
instrument: r.get(5)?,
|
||||
category: r.get(6)?,
|
||||
source: r.get(7)?,
|
||||
reimbursable_to: r.get(8)?,
|
||||
},
|
||||
note: r.get(9)?,
|
||||
})
|
||||
})?;
|
||||
Ok(rows.collect::<Result<_, _>>()?)
|
||||
}
|
||||
|
||||
/// The base currency of a journal: its first line's, which must be a
|
||||
/// sound `store.created` ([`created_currency`]).
|
||||
pub(crate) fn journal_currency(lines: &[JournalLine]) -> Result<Currency, StoreError> {
|
||||
|
||||
@@ -6,3 +6,4 @@
|
||||
# everyone who runs the test benefits from these saved cases.
|
||||
cc 704ab49c9bea84359475c3afa56c3354275a353670fc71156fba151f846f4f73 # shrinks to first = [Create(false, 0, 1), Create(false, 0, 1)], then = [Close(1), Hand(false, 1)]
|
||||
cc f678814ec34db3abef40116e17e562fe5abe652c09f12600e19841beaf9d6fe6 # shrinks to first = [Create(false, 0, 13902), Settle { a: 0, b: 0, n: 0, cash: 6676, discount: 551, fee: 0 }], then = []
|
||||
cc fade94264cb4e769ef3d72573baff01fc2f78483a32e8424e56251d96c4af57e # shrinks to first = [Create(true, 0, 27747), WriteOff(0, 0, 1), Create(false, 0, 1)], then = [Create(false, 0, 7), WriteOff(3, 66, 2761)]
|
||||
@@ -912,6 +912,9 @@ fn issued_pdf_is_filed_with_the_invoice_or_nothing_is() {
|
||||
"entry.posted",
|
||||
"invoice.created",
|
||||
"receipt.attached",
|
||||
// P6: a filed PDF's text is read at attach time in books that
|
||||
// declared `attachment_text`, journaled right after the filing.
|
||||
"attachment.text_extracted",
|
||||
"attachment.linked",
|
||||
]
|
||||
);
|
||||
@@ -925,7 +928,7 @@ fn issued_pdf_is_filed_with_the_invoice_or_nothing_is() {
|
||||
assert_eq!(got[5].1["description"], json!("Issued invoice"));
|
||||
assert_eq!(got[5].1["size_bytes"], json!(pdf.len() as u64));
|
||||
assert_eq!(
|
||||
got[6].1["subject"],
|
||||
got[7].1["subject"],
|
||||
json!({"kind": "invoice", "id": inv}),
|
||||
"the copy is linked to the invoice"
|
||||
);
|
||||
|
||||
@@ -1452,3 +1452,319 @@ fn a_file_linked_to_an_entry_cannot_also_be_filed_on_it() {
|
||||
s.link_receipt(a, fx.e3).unwrap();
|
||||
s.verify().unwrap();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- evidence (P6)
|
||||
|
||||
use ledger_store::format::{ATTACHMENT_TEXT, JournalFormat, current};
|
||||
use ledger_store::{BlobStore, ExtractionStatus, JournalWriter};
|
||||
|
||||
/// Plan P6 step 2: reading a file's text is journaled once per extractor
|
||||
/// and version; the text is a content-addressed blob; `verify` checks it.
|
||||
#[test]
|
||||
fn extraction_is_journaled_once_per_version() {
|
||||
let dir = test_dir("extract-once");
|
||||
let (mut s, fx) = fixture(&dir);
|
||||
let text = b"Zelle payment to Harbor Design, 12.34, 2026-02-10\n";
|
||||
let (id, _, _) = s
|
||||
.attach_receipt(Some(fx.e2), "zelle.txt", "text/plain", text, None)
|
||||
.unwrap();
|
||||
let x = s.extraction(id).unwrap().expect("an extraction record");
|
||||
assert_eq!(x.status, ExtractionStatus::Ok);
|
||||
assert_eq!(x.pages, 1);
|
||||
assert_eq!(x.extractor, ledger_import::extract::EXTRACTOR);
|
||||
assert_eq!(x.version, ledger_import::extract::VERSION);
|
||||
assert_eq!(
|
||||
s.extraction_text(id).unwrap().as_deref(),
|
||||
Some(String::from_utf8_lossy(text).as_ref())
|
||||
);
|
||||
// One record: journaled once, and the text is a stored blob.
|
||||
let records = s
|
||||
.extractions()
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.filter(|x| x.attachment_id == id)
|
||||
.count();
|
||||
assert_eq!(records, 1);
|
||||
let sha = x.text_sha256.expect("text");
|
||||
assert!(s.stored_blob_shas().unwrap().contains(&sha));
|
||||
let seq = s.journal_seq();
|
||||
let again = s.extract_attachment(id).unwrap();
|
||||
assert_eq!(again.seq, x.seq, "the same record");
|
||||
assert_eq!(s.journal_seq(), seq, "a repeat journals nothing");
|
||||
s.verify().unwrap();
|
||||
// And it survives a reopen: the record and the text are the journal's.
|
||||
drop(s);
|
||||
let s = Store::open(&dir).unwrap();
|
||||
assert_eq!(
|
||||
s.extraction_text(id).unwrap().as_deref(),
|
||||
Some("Zelle payment to Harbor Design, 12.34, 2026-02-10\n")
|
||||
);
|
||||
assert_eq!(s.extractions().unwrap().len(), 1);
|
||||
s.verify().unwrap();
|
||||
}
|
||||
|
||||
/// Plan P6 step 2, R-A: reading a file's text waits for the books to
|
||||
/// declare `attachment_text`, journaling nothing until then; attaching a
|
||||
/// file works on old books (the extraction is the backfill's job).
|
||||
#[test]
|
||||
fn extract_waits_for_attachment_text() {
|
||||
let dir = test_dir("extract-upgrade");
|
||||
let mut s = Store::create_in_format(&dir, Currency::USD, &JournalFormat::default()).unwrap();
|
||||
let (id, _, _) = s
|
||||
.attach_receipt(None, "note.txt", "text/plain", b"a note", None)
|
||||
.unwrap();
|
||||
assert!(s.extractions().unwrap().is_empty(), "no feature, no record");
|
||||
let head = journal_bytes(&dir);
|
||||
match s.extract_attachment(id) {
|
||||
Err(StoreError::FormatUpgradePending { need }) => {
|
||||
assert_eq!(need, format!("the journal feature {ATTACHMENT_TEXT}"))
|
||||
}
|
||||
other => panic!("expected FormatUpgradePending, got {other:?}"),
|
||||
}
|
||||
assert_eq!(journal_bytes(&dir), head, "a refusal journals nothing");
|
||||
s.upgrade_format().unwrap();
|
||||
assert_eq!(s.journal_format(), ¤t());
|
||||
let x = s.extract_attachment(id).unwrap();
|
||||
assert_eq!(x.status, ExtractionStatus::Ok);
|
||||
assert_eq!(s.extraction_text(id).unwrap().as_deref(), Some("a note"));
|
||||
// The books replay as written, live and in the projection.
|
||||
s.verify().unwrap();
|
||||
drop(s);
|
||||
let s = Store::open(&dir).unwrap();
|
||||
assert_eq!(s.extraction_text(id).unwrap().as_deref(), Some("a note"));
|
||||
s.verify().unwrap();
|
||||
}
|
||||
|
||||
// Plan P6 step 4: identical bytes are one live attachment, linked to
|
||||
// every subject they support (property: for any number of filings).
|
||||
proptest::proptest! {
|
||||
// Each case makes books on disk (fsync per line): eight is a property
|
||||
// over the count, not a bulk run.
|
||||
#![proptest_config(proptest::prelude::ProptestConfig::with_cases(8))]
|
||||
|
||||
#[test]
|
||||
fn same_bytes_one_record_many_links(n in 1usize..=4) {
|
||||
let dir = test_dir(&format!("dedup-{n}"));
|
||||
let (mut s, fx) = fixture(&dir);
|
||||
let entries = [fx.e1, fx.e2, fx.e3, fx.e4];
|
||||
let bytes = b"one receipt, four takers";
|
||||
let mut ids = Vec::new();
|
||||
for e in entries.iter().take(n) {
|
||||
let (id, sha, dedup) = s
|
||||
.attach_receipt(Some(*e), "r.txt", "text/plain", bytes, None)
|
||||
.unwrap();
|
||||
proptest::prop_assert_eq!(
|
||||
sha,
|
||||
ledger_store::journal::sha256_hex(bytes)
|
||||
);
|
||||
proptest::prop_assert!(dedup || ids.is_empty());
|
||||
ids.push(id);
|
||||
}
|
||||
let all: Vec<Attachment> = s.list_attachments(None).unwrap();
|
||||
proptest::prop_assert_eq!(all.len(), 1, "one record");
|
||||
proptest::prop_assert!(ids.iter().all(|id| *id == all[0].id));
|
||||
let links = s.attachment_links(all[0].id).unwrap();
|
||||
proptest::prop_assert_eq!(links.len(), n);
|
||||
proptest::prop_assert_eq!(
|
||||
s.extractions().unwrap().len(),
|
||||
1,
|
||||
"extraction once"
|
||||
);
|
||||
s.verify().unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
/// Plan P6 step 4: records an older version filed twice open as written;
|
||||
/// withdrawing one as a duplicate records what it duplicates.
|
||||
#[test]
|
||||
fn legacy_duplicates_replay_as_written() {
|
||||
let dir = test_dir("legacy-dupes");
|
||||
let s = Store::create(&dir, Currency::USD).unwrap();
|
||||
drop(s);
|
||||
let bytes = b"filed twice by an older version";
|
||||
let (sha, _) = BlobStore::open(&dir.join("blobs"))
|
||||
.unwrap()
|
||||
.put(bytes)
|
||||
.unwrap();
|
||||
let mut w = JournalWriter::open(&dir.join(JOURNAL_FILE)).unwrap();
|
||||
for (id, name) in [(1, "one.pdf"), (2, "two.pdf")] {
|
||||
w.append(
|
||||
"receipt.attached",
|
||||
json!({
|
||||
"attachment_id": id, "blob_sha": sha, "entry_id": null,
|
||||
"category": "receipts", "file_name": name,
|
||||
"mime_type": "application/pdf", "size_bytes": bytes.len() as u64,
|
||||
"description": null,
|
||||
}),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
drop(w);
|
||||
let mut s = Store::open(&dir).unwrap();
|
||||
let all = s.list_attachments(None).unwrap();
|
||||
assert_eq!(all.len(), 2, "both, as written");
|
||||
assert_eq!(all[0].blob_sha, all[1].blob_sha);
|
||||
assert!(all.iter().all(|a| a.duplicate_of.is_none()));
|
||||
s.withdraw_attachment_duplicate(2, "duplicate of #1", 1)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
s.attachment_by_id(2).unwrap().unwrap().duplicate_of,
|
||||
Some(1)
|
||||
);
|
||||
s.verify().unwrap();
|
||||
drop(s);
|
||||
let s = Store::open(&dir).unwrap();
|
||||
let again = s.attachment_by_id(2).unwrap().unwrap();
|
||||
assert_eq!(again.duplicate_of, Some(1));
|
||||
assert!(again.withdrawn_at.is_some());
|
||||
s.verify().unwrap();
|
||||
}
|
||||
|
||||
/// Plan P6 step 5: annotations replay and project, replacing an earlier
|
||||
/// one, and the engine's next receipt number runs from the highest.
|
||||
#[test]
|
||||
fn annotations_replay_and_project() {
|
||||
let dir = test_dir("annotations");
|
||||
let (mut s, fx) = fixture(&dir);
|
||||
let (id, _, _) = s
|
||||
.attach_receipt(Some(fx.e2), "r.pdf", "application/pdf", b"%PDF r", None)
|
||||
.unwrap();
|
||||
let first = ledger_core::Annotation {
|
||||
attachment_id: id,
|
||||
fields: ledger_core::AnnotationFields {
|
||||
receipt_number: Some(4),
|
||||
payee: Some("Harbor Design".into()),
|
||||
amount: Some(dec("12.34")),
|
||||
date: Some(date(2026, 2, 1)),
|
||||
instrument: Some("Check 1234".into()),
|
||||
category: None,
|
||||
source: Some("the owner's email".into()),
|
||||
reimbursable_to: Some("Avery Quill".into()),
|
||||
},
|
||||
note: Some("read on 2026-02-10".into()),
|
||||
};
|
||||
s.annotate_attachment(first.clone()).unwrap();
|
||||
assert_eq!(s.ledger().annotation(id), Some(&first), "the live ledger");
|
||||
assert_eq!(s.annotations().unwrap(), vec![first.clone()]);
|
||||
assert_eq!(s.ledger().next_receipt_number(), 5);
|
||||
// A later annotation replaces it (one projection row, one ledger map
|
||||
// entry).
|
||||
let second = ledger_core::Annotation {
|
||||
attachment_id: id,
|
||||
fields: ledger_core::AnnotationFields {
|
||||
receipt_number: Some(6),
|
||||
payee: Some("Harbor Design LLC".into()),
|
||||
..Default::default()
|
||||
},
|
||||
note: None,
|
||||
};
|
||||
s.annotate_attachment(second.clone()).unwrap();
|
||||
assert_eq!(s.annotations().unwrap(), vec![second.clone()]);
|
||||
assert_eq!(s.ledger().next_receipt_number(), 7);
|
||||
s.verify().unwrap();
|
||||
drop(s);
|
||||
let s = Store::open(&dir).unwrap();
|
||||
assert_eq!(s.ledger().annotation(id), Some(&second));
|
||||
assert_eq!(s.annotations().unwrap(), vec![second]);
|
||||
s.verify().unwrap();
|
||||
}
|
||||
|
||||
/// Plan P6 step 5: receipt numbers run from the maximum annotated.
|
||||
#[test]
|
||||
fn receipt_numbers_run_from_the_max() {
|
||||
let dir = test_dir("receipt-numbers");
|
||||
let (mut s, fx) = fixture(&dir);
|
||||
let price = |n: u32| ledger_core::Annotation {
|
||||
attachment_id: 0,
|
||||
fields: ledger_core::AnnotationFields {
|
||||
receipt_number: Some(n),
|
||||
..Default::default()
|
||||
},
|
||||
note: None,
|
||||
};
|
||||
assert_eq!(s.ledger().next_receipt_number(), 1, "nothing annotated");
|
||||
let ids: Vec<u64> = (0..3)
|
||||
.map(|i| {
|
||||
s.attach_receipt(
|
||||
Some([fx.e1, fx.e2, fx.e3][i]),
|
||||
&format!("r{i}.txt"),
|
||||
"text/plain",
|
||||
format!("receipt {i}").as_bytes(),
|
||||
None,
|
||||
)
|
||||
.unwrap()
|
||||
.0
|
||||
})
|
||||
.collect();
|
||||
for (id, (n, next)) in ids.iter().zip([(3, 4), (9, 10), (7, 10)]) {
|
||||
let mut a = price(n);
|
||||
a.attachment_id = *id;
|
||||
s.annotate_attachment(a).unwrap();
|
||||
assert_eq!(s.ledger().next_receipt_number(), next, "{n}");
|
||||
}
|
||||
assert_eq!(s.ledger().next_receipt_number(), 10);
|
||||
s.verify().unwrap();
|
||||
}
|
||||
|
||||
/// Plan P6 step 5: "reimbursed" is derived from `entry.settles` only;
|
||||
/// nothing about a member-paid entry's shape makes it paid back.
|
||||
#[test]
|
||||
fn reimbursed_is_derived_from_settles_only() {
|
||||
let dir = test_dir("settles");
|
||||
let mut s = Store::create(&dir, Currency::USD).unwrap();
|
||||
let cash = s
|
||||
.add_account("1000", "Cash", AccountType::Asset, None, false)
|
||||
.unwrap();
|
||||
let due = s
|
||||
.add_account(
|
||||
"2100",
|
||||
"Due to Members",
|
||||
AccountType::Liability,
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
let expense = s
|
||||
.add_account("6000", "Hosting", AccountType::Expense, None, false)
|
||||
.unwrap();
|
||||
let paid = post(
|
||||
&mut s,
|
||||
date(2026, 2, 1),
|
||||
"Hosting, paid by a member",
|
||||
expense,
|
||||
due,
|
||||
"20.00",
|
||||
);
|
||||
let reimb = post(
|
||||
&mut s,
|
||||
date(2026, 2, 10),
|
||||
"Reimburse the member",
|
||||
due,
|
||||
cash,
|
||||
"20.00",
|
||||
);
|
||||
assert!(!s.ledger().settles_entry(paid), "no record yet");
|
||||
assert!(s.ledger().entry_settles(reimb).is_empty());
|
||||
s.set_entry_settles(reimb, &[paid]).unwrap();
|
||||
assert!(s.ledger().settles_entry(paid));
|
||||
assert_eq!(s.ledger().entry_settles(reimb), [paid]);
|
||||
// Refusals journal nothing and change no record.
|
||||
let head = journal_bytes(&dir);
|
||||
for (entry, settles) in [
|
||||
(paid, vec![paid]),
|
||||
(reimb, vec![paid, paid]),
|
||||
(reimb, vec![999]),
|
||||
] {
|
||||
assert!(
|
||||
s.set_entry_settles(entry, &settles).is_err(),
|
||||
"{entry} {settles:?}"
|
||||
);
|
||||
}
|
||||
assert_eq!(journal_bytes(&dir), head, "refusals journal nothing");
|
||||
s.verify().unwrap();
|
||||
drop(s);
|
||||
let s = Store::open(&dir).unwrap();
|
||||
assert!(s.ledger().settles_entry(paid));
|
||||
assert_eq!(s.ledger().entry_settles(reimb), [paid]);
|
||||
s.verify().unwrap();
|
||||
}
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
# Records
|
||||
|
||||
Kept with the company's papers.
|
||||
+1
@@ -0,0 +1 @@
|
||||
Zelle payment to A. Quill, 20.00, 2026-02-10
|
||||
BIN
Binary file not shown.
@@ -0,0 +1,187 @@
|
||||
{
|
||||
"assets": [],
|
||||
"bills": [],
|
||||
"entries": [
|
||||
{
|
||||
"date": "2026-01-01",
|
||||
"description": "Opening capital",
|
||||
"id": 1,
|
||||
"lines": [
|
||||
{
|
||||
"account": 1,
|
||||
"amount": {
|
||||
"amount": "500.00",
|
||||
"currency": [
|
||||
85,
|
||||
83,
|
||||
68
|
||||
]
|
||||
},
|
||||
"side": "Debit"
|
||||
},
|
||||
{
|
||||
"account": 3,
|
||||
"amount": {
|
||||
"amount": "500.00",
|
||||
"currency": [
|
||||
85,
|
||||
83,
|
||||
68
|
||||
]
|
||||
},
|
||||
"side": "Credit"
|
||||
}
|
||||
],
|
||||
"reversal_of": null,
|
||||
"status": "Posted"
|
||||
},
|
||||
{
|
||||
"date": "2026-02-01",
|
||||
"description": "Web hosting, paid by a member",
|
||||
"id": 2,
|
||||
"lines": [
|
||||
{
|
||||
"account": 4,
|
||||
"amount": {
|
||||
"amount": "20.00",
|
||||
"currency": [
|
||||
85,
|
||||
83,
|
||||
68
|
||||
]
|
||||
},
|
||||
"side": "Debit"
|
||||
},
|
||||
{
|
||||
"account": 2,
|
||||
"amount": {
|
||||
"amount": "20.00",
|
||||
"currency": [
|
||||
85,
|
||||
83,
|
||||
68
|
||||
]
|
||||
},
|
||||
"side": "Credit"
|
||||
}
|
||||
],
|
||||
"reversal_of": null,
|
||||
"status": "Posted"
|
||||
},
|
||||
{
|
||||
"date": "2026-02-10",
|
||||
"description": "Reimburse the member",
|
||||
"id": 3,
|
||||
"lines": [
|
||||
{
|
||||
"account": 2,
|
||||
"amount": {
|
||||
"amount": "20.00",
|
||||
"currency": [
|
||||
85,
|
||||
83,
|
||||
68
|
||||
]
|
||||
},
|
||||
"side": "Debit"
|
||||
},
|
||||
{
|
||||
"account": 1,
|
||||
"amount": {
|
||||
"amount": "20.00",
|
||||
"currency": [
|
||||
85,
|
||||
83,
|
||||
68
|
||||
]
|
||||
},
|
||||
"side": "Credit"
|
||||
}
|
||||
],
|
||||
"reversal_of": null,
|
||||
"status": "Posted"
|
||||
},
|
||||
{
|
||||
"date": "2026-03-01",
|
||||
"description": "March rent",
|
||||
"id": 4,
|
||||
"lines": [
|
||||
{
|
||||
"account": 4,
|
||||
"amount": {
|
||||
"amount": "100.00",
|
||||
"currency": [
|
||||
85,
|
||||
83,
|
||||
68
|
||||
]
|
||||
},
|
||||
"side": "Debit"
|
||||
},
|
||||
{
|
||||
"account": 1,
|
||||
"amount": {
|
||||
"amount": "100.00",
|
||||
"currency": [
|
||||
85,
|
||||
83,
|
||||
68
|
||||
]
|
||||
},
|
||||
"side": "Credit"
|
||||
}
|
||||
],
|
||||
"reversal_of": null,
|
||||
"status": "Posted"
|
||||
}
|
||||
],
|
||||
"entry_parties": [],
|
||||
"invoices": [],
|
||||
"payroll_runs": [],
|
||||
"periods": [],
|
||||
"recurring": [],
|
||||
"schedules": [],
|
||||
"snapshot": {
|
||||
"Ok": {
|
||||
"balance_sheet": {
|
||||
"assets": "380",
|
||||
"balanced": true,
|
||||
"current_earnings": "-120",
|
||||
"equity": "380",
|
||||
"liabilities": "0"
|
||||
},
|
||||
"profit_and_loss": {
|
||||
"expenses": "120",
|
||||
"net_income": "-120",
|
||||
"revenue": "0"
|
||||
},
|
||||
"trial_balance": {
|
||||
"lines": [
|
||||
{
|
||||
"code": "1000",
|
||||
"credit": "0",
|
||||
"debit": "380",
|
||||
"name": "Cash"
|
||||
},
|
||||
{
|
||||
"code": "3000",
|
||||
"credit": "500",
|
||||
"debit": "0",
|
||||
"name": "Capital"
|
||||
},
|
||||
{
|
||||
"code": "6000",
|
||||
"credit": "0",
|
||||
"debit": "120",
|
||||
"name": "Rent"
|
||||
}
|
||||
],
|
||||
"total_credits": "500",
|
||||
"total_debits": "500"
|
||||
}
|
||||
}
|
||||
},
|
||||
"verify": {
|
||||
"Ok": null
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
{"payload":{"base_currency":"USD"},"prev":"0000000000000000000000000000000000000000000000000000000000000000","seq":1,"ts":"2026-10-27T15:18:09Z","type":"store.created"}
|
||||
{"payload":{"features":["account_renamed","asset_basis","depreciation_catch_up","disposal_before_service","document_line_projects","document_standing","documents_v2","members_cumulative","month_reposting","party_creditors","payment_instruments","projects","require_tax_category","schedule_convention","tax_categories_gain_exempt","vendor_1099_decisions"],"format":2},"prev":"7f124ed3e91db7042661abf4e5007b1dcae9903a767bea76e7e727a96075bfa6","seq":2,"ts":"2026-10-27T15:18:09Z","type":"journal.format_set"}
|
||||
{"payload":{"account_type":"Asset","code":"1000","currency":"USD","id":1,"name":"Cash","parent":null,"placeholder":false},"prev":"955f321d1a0bbd460640ba5339b2d3e1313038f3ba615e5ffff223ba3c8c728f","seq":3,"ts":"2026-10-27T15:18:09Z","type":"account.added"}
|
||||
{"payload":{"account_type":"Liability","code":"2100","currency":"USD","id":2,"name":"Due to Members","parent":null,"placeholder":false},"prev":"7b619dff5f0fb25b516b96b459b39ef67713bef25689946837a28f58e7692db5","seq":4,"ts":"2026-10-27T15:18:09Z","type":"account.added"}
|
||||
{"payload":{"account_type":"Equity","code":"3000","currency":"USD","id":3,"name":"Capital","parent":null,"placeholder":false},"prev":"d55d7fb14bbfbd1aa43f067617901d7086ecbfa16a693f0aa651b98ba525e4f8","seq":5,"ts":"2026-10-27T15:18:09Z","type":"account.added"}
|
||||
{"payload":{"account_type":"Expense","code":"6000","currency":"USD","id":4,"name":"Rent","parent":null,"placeholder":false},"prev":"6cd464539b1b1aa60ae1093728da6a48658cc7bf3f962c88a21c863b85b17523","seq":6,"ts":"2026-10-27T15:18:09Z","type":"account.added"}
|
||||
{"payload":{"date":"2026-01-01","description":"Opening capital","id":1},"prev":"ed316ff3ac6c630502dd56101b39b53fac83e4554fd78e271c91d0d468ae6c93","seq":7,"ts":"2026-10-27T15:18:09Z","type":"entry.created"}
|
||||
{"payload":{"account":1,"amount":"500.00","currency":"USD","id":1,"side":"debit"},"prev":"cf19ca8e1f8992a73c249c3d5e06997b681799a19d39261082d66ecee54a7c78","seq":8,"ts":"2026-10-27T15:18:09Z","type":"entry.line_added"}
|
||||
{"payload":{"account":3,"amount":"500.00","currency":"USD","id":1,"side":"credit"},"prev":"31d98c34346a4ca836f1b7a51e993a7c8b682eaab36f061d92e8659eafcf3691","seq":9,"ts":"2026-10-27T15:18:09Z","type":"entry.line_added"}
|
||||
{"payload":{"date":"2026-01-01","description":"Opening capital","id":1,"lines":[{"account":1,"amount":"500.00","currency":"USD","side":"debit"},{"account":3,"amount":"500.00","currency":"USD","side":"credit"}],"reversal_of":null},"prev":"1589ea4608834aab2019ebd960438eed2ebd843abef29fbcf2174f65b88a34e8","seq":10,"ts":"2026-10-27T15:18:09Z","type":"entry.posted"}
|
||||
{"payload":{"date":"2026-02-01","description":"Web hosting, paid by a member","id":2},"prev":"ea67596a958a0c100bc81e4d4b7058c3145803ea765a3989d3da58eef343c775","seq":11,"ts":"2026-10-27T15:18:09Z","type":"entry.created"}
|
||||
{"payload":{"account":4,"amount":"20.00","currency":"USD","id":2,"side":"debit"},"prev":"1e78ebd8c378a723511d3e054f216c02587d9c0795a2958b64a7ef681405301f","seq":12,"ts":"2026-10-27T15:18:09Z","type":"entry.line_added"}
|
||||
{"payload":{"account":2,"amount":"20.00","currency":"USD","id":2,"side":"credit"},"prev":"06d014884d44765a38cb4ca9220cef0b913ef8a8b561997fdea7d59c0094dc0e","seq":13,"ts":"2026-10-27T15:18:09Z","type":"entry.line_added"}
|
||||
{"payload":{"date":"2026-02-01","description":"Web hosting, paid by a member","id":2,"lines":[{"account":4,"amount":"20.00","currency":"USD","side":"debit"},{"account":2,"amount":"20.00","currency":"USD","side":"credit"}],"reversal_of":null},"prev":"1863ad9b2fb040fd24c1398b7ec995f083153fbacc5e252e88bcd98541e3c33a","seq":14,"ts":"2026-10-27T15:18:10Z","type":"entry.posted"}
|
||||
{"payload":{"date":"2026-02-10","description":"Reimburse the member","id":3},"prev":"6e7ec8d7738b85d7e79548f22eb6e314a370a31757c1651a67fa752b7adc4651","seq":15,"ts":"2026-10-27T15:18:10Z","type":"entry.created"}
|
||||
{"payload":{"account":2,"amount":"20.00","currency":"USD","id":3,"side":"debit"},"prev":"a26929387cbba06f0e89c9b1dd4118320f22be51d8ee499ee398e411f16b890f","seq":16,"ts":"2026-10-27T15:18:10Z","type":"entry.line_added"}
|
||||
{"payload":{"account":1,"amount":"20.00","currency":"USD","id":3,"side":"credit"},"prev":"b2e9405890eb89ae0e5afd17d3e5cf741e330960f3f01449bd7a82c2e1dea440","seq":17,"ts":"2026-10-27T15:18:10Z","type":"entry.line_added"}
|
||||
{"payload":{"date":"2026-02-10","description":"Reimburse the member","id":3,"lines":[{"account":2,"amount":"20.00","currency":"USD","side":"debit"},{"account":1,"amount":"20.00","currency":"USD","side":"credit"}],"reversal_of":null},"prev":"94f2dbba114035ebe433c985c6b4f5ace26bd4588903fff15bef8717ab82bc77","seq":18,"ts":"2026-10-27T15:18:10Z","type":"entry.posted"}
|
||||
{"payload":{"date":"2026-03-01","description":"March rent","id":4},"prev":"b1c37b5614c19fdbe10e16ded9072b31bc0f2576cd6419a86c14d258871ba6dc","seq":19,"ts":"2026-10-27T15:18:10Z","type":"entry.created"}
|
||||
{"payload":{"account":4,"amount":"100.00","currency":"USD","id":4,"side":"debit"},"prev":"66f50f061925b1be9ad46a015ab7359729324b58419ee97abdc2e430c19cd335","seq":20,"ts":"2026-10-27T15:18:10Z","type":"entry.line_added"}
|
||||
{"payload":{"account":1,"amount":"100.00","currency":"USD","id":4,"side":"credit"},"prev":"1e5e7bd1e14131c980d43b5da0fb6a21673dd3f7120dfe87873cd7e88eb49340","seq":21,"ts":"2026-10-27T15:18:10Z","type":"entry.line_added"}
|
||||
{"payload":{"date":"2026-03-01","description":"March rent","id":4,"lines":[{"account":4,"amount":"100.00","currency":"USD","side":"debit"},{"account":1,"amount":"100.00","currency":"USD","side":"credit"}],"reversal_of":null},"prev":"c3ee5fc9f8cf08b452969df15921f6bcaf61514c608971b6180e8b89bdbabfbb","seq":22,"ts":"2026-10-27T15:18:10Z","type":"entry.posted"}
|
||||
{"payload":{"attachment_id":1,"blob_sha":"5837c77f55b72825a8a6be69e2aecc3e962fba90277e12547ae2d6b7f1ad0435","category":"receipts","description":"March rent invoice","entry_id":4,"file_name":"receipt.pdf","mime_type":"application/pdf","size_bytes":20748},"prev":"4820680a8b8f00fbfee12dcdb515f6c7232fe002b17f6a0d875cbc7b37f9301c","seq":23,"ts":"2026-10-27T15:18:10Z","type":"receipt.attached"}
|
||||
{"payload":{"attachment_id":2,"blob_sha":"2d6666a14d257ebe2f29c5a10380afaafed93efee2b80d79af3d3d53c10af699","category":"receipts","description":null,"entry_id":3,"file_name":"zelle.txt","mime_type":"text/plain","size_bytes":45},"prev":"06043343802dce570e36d0502cc359bc05dd00f5ccf0831f34c516648ee0c9dc","seq":24,"ts":"2026-10-27T15:18:10Z","type":"receipt.attached"}
|
||||
{"payload":{"attachment_id":3,"blob_sha":"1697fa59fa7f1f7e6d4e3b53940e9208596242b8451ddb01bef8b00b71040c2b","category":"other","description":null,"entry_id":null,"file_name":"note.md","mime_type":"text/markdown","size_bytes":43},"prev":"9cad59cd4384dc0769bc925032cad7366c814e827d094795404b9b6d9e5ce861","seq":25,"ts":"2026-10-27T15:18:10Z","type":"document.filed"}
|
||||
@@ -940,11 +940,14 @@ fn attach_stores_under_sha256() {
|
||||
assert_eq!(b.store.verify_blobs().unwrap(), 1);
|
||||
}
|
||||
|
||||
/// Oracle: Store::attach_receipt INVARIANTS 2 and 3 — the same bytes filed
|
||||
/// twice share one stored file, the second says deduplicated, and each
|
||||
/// filing is its own journaled attachment with its name, type, description.
|
||||
/// Oracle: Store::attach_receipt INVARIANTS 2 and 3, as plan P6 step 4
|
||||
/// changes them — the same bytes filed twice share one stored file and one
|
||||
/// live attachment, the second says deduplicated, and that attachment is
|
||||
/// linked to the second subject (its name, type and description are the
|
||||
/// first filing's). Records an older version filed twice stay as written
|
||||
/// (`legacy_duplicates_replay_as_written`).
|
||||
#[test]
|
||||
fn attach_dedups_bytes_but_not_attachments() {
|
||||
fn same_bytes_are_one_attachment_linked_to_both() {
|
||||
let dir = test_dir("attach-dedup");
|
||||
let mut b = books(&dir);
|
||||
let bytes = b"same bytes";
|
||||
@@ -962,52 +965,61 @@ fn attach_dedups_bytes_but_not_attachments() {
|
||||
.store
|
||||
.attach_receipt(Some(b.e2), "two.png", "image/png", bytes, Some("second"))
|
||||
.unwrap();
|
||||
assert_eq!((a1, a2), (a1, a1), "one attachment");
|
||||
assert_eq!(s1, s2);
|
||||
assert!(!d1);
|
||||
assert!(d2);
|
||||
assert_ne!(a1, a2);
|
||||
let blob_files: Vec<_> = tree(&dir)
|
||||
.into_keys()
|
||||
.filter(|p| p.starts_with("blobs"))
|
||||
.collect();
|
||||
assert_eq!(blob_files.len(), 1, "one stored file: {blob_files:?}");
|
||||
let all = b.store.list_attachments(None).unwrap();
|
||||
let one = all.iter().find(|a| a.id == a1).expect("first listed");
|
||||
let two = all.iter().find(|a| a.id == a2).expect("second listed");
|
||||
assert_eq!(all.len(), 1, "one record: {all:?}");
|
||||
let one = all.iter().find(|a| a.id == a1).expect("listed");
|
||||
assert_eq!(one.entry_id, Some(b.e1));
|
||||
assert_eq!(two.entry_id, Some(b.e2));
|
||||
assert_eq!(
|
||||
(one.file_name.as_str(), one.mime_type.as_str()),
|
||||
("one.pdf", "application/pdf")
|
||||
);
|
||||
assert_eq!(one.description.as_deref(), Some("first"));
|
||||
assert_eq!(two.description.as_deref(), Some("second"));
|
||||
assert_eq!(one.size_bytes, bytes.len() as u64);
|
||||
assert_eq!(one.blob_sha, s1);
|
||||
// The second subject is linked, and counts as its receipt.
|
||||
assert!(
|
||||
b.store
|
||||
.attachment_links(a1)
|
||||
.unwrap()
|
||||
.contains(&ledger_store::Subject::Entry(b.e2))
|
||||
);
|
||||
assert_eq!(
|
||||
b.store
|
||||
.attachments_for(ledger_store::Subject::Entry(b.e2))
|
||||
.unwrap()
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
// One filing line: the second added a link, not a record.
|
||||
let attached: Vec<Value> = parsed_lines(&dir)
|
||||
.into_iter()
|
||||
.filter(|l| l["type"] == "receipt.attached")
|
||||
.collect();
|
||||
assert_eq!(attached.len(), 2);
|
||||
for (l, (name, mime, desc)) in attached.iter().zip([
|
||||
("one.pdf", "application/pdf", "first"),
|
||||
("two.png", "image/png", "second"),
|
||||
]) {
|
||||
for needle in [name, mime, desc, s1.as_str()] {
|
||||
assert!(
|
||||
contains_value(&l["payload"], needle),
|
||||
"{needle} missing from {l}"
|
||||
);
|
||||
}
|
||||
assert_eq!(attached.len(), 1);
|
||||
for needle in ["one.pdf", "application/pdf", "first", s1.as_str()] {
|
||||
assert!(
|
||||
contains_value(&attached[0]["payload"], needle),
|
||||
"{needle} missing from {}",
|
||||
attached[0]
|
||||
);
|
||||
}
|
||||
assert_eq!(b.store.attachment_bytes(a2).unwrap(), bytes);
|
||||
assert_eq!(b.store.attachment_bytes(a1).unwrap(), bytes);
|
||||
}
|
||||
|
||||
/// Oracle: Store::attach_receipt INVARIANT 3 — each filing is its own
|
||||
/// attachment with ITS name and type, even when its bytes deduplicate
|
||||
/// against an earlier filing.
|
||||
/// Oracle: Store::attach_receipt INVARIANT 3, as plan P6 step 4 changes
|
||||
/// it — one file, one record: the first filing's own name and type stay,
|
||||
/// in the writer and in the viewer.
|
||||
#[test]
|
||||
fn attach_dedup_keeps_each_filings_name_and_type() {
|
||||
fn the_first_filings_name_and_type_stay() {
|
||||
let dir = test_dir("attach-dedup-name");
|
||||
let mut b = books(&dir);
|
||||
let bytes = b"same bytes, two names";
|
||||
@@ -1019,28 +1031,25 @@ fn attach_dedup_keeps_each_filings_name_and_type() {
|
||||
.store
|
||||
.attach_receipt(Some(b.e2), "two.png", "image/png", bytes, None)
|
||||
.unwrap();
|
||||
assert_eq!((a1, a2), (a1, a1));
|
||||
assert!(d2);
|
||||
let all = b.store.list_attachments(None).unwrap();
|
||||
assert_eq!(all.len(), 1);
|
||||
let one = all.iter().find(|a| a.id == a1).unwrap();
|
||||
let two = all.iter().find(|a| a.id == a2).unwrap();
|
||||
assert_eq!(
|
||||
(one.file_name.as_str(), one.mime_type.as_str()),
|
||||
("one.pdf", "application/pdf")
|
||||
);
|
||||
assert_eq!(
|
||||
(two.file_name.as_str(), two.mime_type.as_str()),
|
||||
("two.png", "image/png")
|
||||
);
|
||||
let snap = Snapshot::load(&dir).unwrap();
|
||||
let s2 = snap
|
||||
let s1 = snap
|
||||
.attachments(None)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.find(|a| a.id == a2)
|
||||
.find(|a| a.id == a1)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
(s2.file_name.as_str(), s2.mime_type.as_str()),
|
||||
("two.png", "image/png")
|
||||
(s1.file_name.as_str(), s1.mime_type.as_str()),
|
||||
("one.pdf", "application/pdf")
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -1893,6 +1893,21 @@ fn books_written_under_older_rules_still_open() {
|
||||
json!({"id": 1, "name": "Tobin Harlow Supply"}),
|
||||
)
|
||||
.unwrap();
|
||||
// P6's new shapes, hand-written as a later version writes them: a file's
|
||||
// extraction record, an annotation of it, and a settlement. This reader
|
||||
// takes them; the version before it stopped at the declaration.
|
||||
w.append(
|
||||
"attachment.text_extracted",
|
||||
json!({"attachment_id": 5, "text_sha256": null, "extractor": "wanderbooks",
|
||||
"version": "1", "pages": 0, "status": "none"}),
|
||||
)
|
||||
.unwrap();
|
||||
w.append(
|
||||
"attachment.annotated",
|
||||
json!({"attachment_id": 5, "fields": {"receipt_number": 3, "payee": "Acme"},
|
||||
"note": "read by hand"}),
|
||||
)
|
||||
.unwrap();
|
||||
w.append(
|
||||
"entry.created",
|
||||
json!({"id": 2, "date": "2026-04-01", "description": "Invoice 1"}),
|
||||
@@ -2070,6 +2085,30 @@ fn books_written_under_older_rules_still_open() {
|
||||
"amount": "30.00", "reason": "uncollectible"}),
|
||||
)
|
||||
.unwrap();
|
||||
// P6's new shapes, hand-written as a later version writes them: an
|
||||
// entry a member was reimbursed by, and the record of what it settles.
|
||||
w.append(
|
||||
"entry.created",
|
||||
json!({"id": 7, "date": "2026-03-02", "description": "Reimburse"}),
|
||||
)
|
||||
.unwrap();
|
||||
for side in ["debit", "credit"] {
|
||||
w.append(
|
||||
"entry.line_added",
|
||||
json!({"id": 7, "account": 2, "side": side, "amount": "5.00", "currency": "USD"}),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
w.append(
|
||||
"entry.posted",
|
||||
json!({"id": 7, "date": "2026-03-02", "description": "Reimburse",
|
||||
"lines": [{"account": 2, "side": "debit", "amount": "5.00", "currency": "USD"},
|
||||
{"account": 2, "side": "credit", "amount": "5.00", "currency": "USD"}],
|
||||
"reversal_of": null}),
|
||||
)
|
||||
.unwrap();
|
||||
w.append("entry.settles", json!({"entry_id": 7, "settles": [1]}))
|
||||
.unwrap();
|
||||
drop(w);
|
||||
// Opening rebuilds from the journal; it must accept all of it.
|
||||
let _ = std::fs::remove_file(dir.join(ledger_store::db_file()));
|
||||
@@ -2086,6 +2125,21 @@ fn books_written_under_older_rules_still_open() {
|
||||
"108.34567".parse::<Decimal>().unwrap()
|
||||
);
|
||||
// The new shapes replay as written: the party's facts and the setting.
|
||||
// P6's shapes replayed: the extraction, the annotation and the
|
||||
// settlement.
|
||||
let extractions = store.extractions().unwrap();
|
||||
assert_eq!(extractions.len(), 1);
|
||||
assert_eq!(extractions[0].attachment_id, 5);
|
||||
assert_eq!(extractions[0].status, ledger_store::ExtractionStatus::None);
|
||||
let annotation = store
|
||||
.ledger()
|
||||
.annotation(5)
|
||||
.expect("the annotation, replayed");
|
||||
assert_eq!(annotation.fields.receipt_number, Some(3));
|
||||
assert_eq!(annotation.fields.payee.as_deref(), Some("Acme"));
|
||||
assert_eq!(store.ledger().next_receipt_number(), 4);
|
||||
assert_eq!(store.ledger().entry_settles(7), [1]);
|
||||
assert!(store.ledger().settles_entry(1));
|
||||
let acme = store.ledger().get_party(1).expect("the party");
|
||||
assert_eq!(acme.email.as_deref(), Some("billing@acme.example"));
|
||||
assert_eq!(acme.phone.as_deref(), Some("555-0142"));
|
||||
|
||||
Reference in new issue
Block a user