From e78ddd59af69d7c34b3ed8fcd1e125fe21db1097 Mon Sep 17 00:00:00 2001 From: Haakam Aujla Date: Fri, 18 Sep 2026 06:06:05 +0000 Subject: [PATCH 1/3] Remove a dead constant and ground the rest in measurement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to #21, which named the remaining constants but left several asserted rather than justified. Each one here was probed: perturb the value, run the suite, and diff the extracted text over a 161-message corpus captured from the tests. Zero corpus outputs change across this commit. Removed: - DETECT_MIN_CONFIDENCE could not affect anything. The gated branch and the fallback under it both returned `detected.encoding`, so every value of the threshold produced identical output — confirmed by enumerating the truth table and by the suite passing at 0, 0.4, 0.9, 0.999 and 2. Deleted, with the ungated behaviour stated where the gate used to be. - The three HTML marker windows (200 / 250 / 80) are one idea, and the closing tag lookahead, not the number, is what confines a marker to its block. Now a single HTML_MARKER_SPAN. The widest real span in the corpus is 184 characters (Outlook's inline CSS between "From:" and the address); all three windows are identical on every corpus input at any shared value from 200 to 400. The 80 was the least grounded: it fires on one input, with a 9-character match. Corrected: - The DETECT_MIN_CONFIDENCE rationale added in #21 was wrong. It claimed the bar was high enough to fall back to latin1 on an ambiguous file; that never happened, because the fallback returned the encoding the gate had rejected. - MAX_PDF_PAGES said it bounds parse work. The per-page deadline check already does that. What the cap actually buys is determinism — a declared page count is a fixed property of the file, so the same attachment extracts identically on a fast and a slow host. Restated; the value is unchanged. Named and justified with their measured bounds — the signature-rescue thresholds flagged as follow-up in #21: - QUOTED_BLOCK_RATIO (0.5) and SIG_VISIBLE_CHARS (400) are the two with a demonstrated bound on BOTH sides, so each comment now cites the case that fails below it and the case that fails above it. - SIG_BODY_LINES (2) and MORE_QUOTE_SPAN (200) are floors; nothing in the corpus constrains them from above, and the comments say so. - The three line lookaheads had an upper-bound argument that measurement does not support: raising ATTRIBUTION_WRAP_LINES to 50, DIVIDER_LOOKAHEAD_LINES to 1000 or HEADER_BLOCK_LINES to 1000 changes no output. Reworded to present the low side as evidence and the high side as judgement. Also recorded where a constant is pinned by a test that does not import it (STREAM_SLICE_UNITS, which fails at both 64 and 1 GB), and where a value has no test coverage at all (SNIFF_TEXT_RATIO passes at 0 through 0.95; DETECT_SAMPLE_BYTES passes at 4 bytes and at 1 GB). Co-Authored-By: Claude Opus 5 (1M context) --- attachment.ts | 58 ++++++++++++++++--------------- body.ts | 96 ++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 105 insertions(+), 49 deletions(-) diff --git a/attachment.ts b/attachment.ts index b4e2434..b3699ad 100644 --- a/attachment.ts +++ b/attachment.ts @@ -35,13 +35,15 @@ const CAP_STORAGE_SLACK = 2 // here: a separator that disagrees with its own charged width makes the cap off by that difference. const BLOCK_SEPARATOR = '\n\n' -// A PDF can declare an enormous page count. Bounds parse work when per-page text is too sparse to -// trip the output cap; a content-bearing PDF hits MAX_OUTPUT_CHARS within a few dozen pages first. -// 2000 is a backstop, not a tuned threshold, and only the sparse case can ever reach it: at even 125 -// characters per page the output cap binds first, so a document this bound is one carrying almost no -// text at all — where the pages parsed, not the text kept, is the cost. Sized against the real -// corpus (an email attachment running past a few hundred pages is already an outlier) with an order -// of magnitude of headroom, so no plausible attachment is cut by it before the deadline intervenes. +// A PDF can declare an enormous page count. What this buys is DETERMINISM, not work-bounding: the +// page loop already checks the deadline before every page, so a pathological page count is bounded +// in time with or without this. Without it, though, where a huge sparse PDF stops would depend on +// how fast the host is, and the same attachment would extract differently on a re-index. A declared +// page count is a fixed property of the file, so cutting on it cuts at the same place every time. +// Only the sparse case can reach it: at even 125 characters per page MAX_OUTPUT_CHARS binds first, +// so a document stopped here is one carrying almost no text at all. 2000 is a backstop rather than +// a tuned threshold — an email attachment past a few hundred pages is already an outlier, and the +// suite is indifferent to raising it (it passes with this at 1e9), so it is sized for headroom. export const MAX_PDF_PAGES = 2000 // Max ACTUAL uncompressed size of an OOXML zip, measured by inflating it — the declared size is @@ -75,6 +77,10 @@ export const HANDLER_DEADLINE_MARGIN_MS = 1_000 // interrupted at all, and one character at a time pays per-slice overhead on every byte. 16 KB is // the compromise, and is chosen to match what zlib pushes out of createInflateRaw so a stored part // and a deflated one arrive in comparable pieces rather than on two different response latencies. +// Pinned from BOTH sides by 'checks the deadline between chunks of a stored document part', which +// sizes its fixture so that the first slice holds the prefix and 5,000 paragraphs do not fit in one: +// the suite fails at 64 and again at 1 GB. That test states 16 KiB in prose rather than importing +// this, so retuning it here surfaces as an unrelated-looking deadline assertion — change both. const STREAM_SLICE_UNITS = 16 * 1024 // saxes resolves namespace prefixes by scanning the open-tag stack, so attacker-controlled nesting @@ -105,17 +111,18 @@ const SNIFF_BYTES = 8 * 1024 // printable (the exceptions are stray control bytes, a handful per file), and binary that gets this // far is ~30-50% by the structure of byte values, since only 95 of 256 are printable ASCII. Anything // from roughly 0.6 to 0.95 separates them identically; 0.85 leaves room for a text file carrying -// some control noise without letting through anything with real binary density. +// some control noise without letting through anything with real binary density. Worth knowing when +// changing it: the NUL check above is doing the discrimination the suite actually exercises, and no +// test distinguishes this value — it passes at 0, 0.5, 0.7 and 0.95, and fails only when set above 1 +// (unreachable, so every input is binary). A file that is, say, 60% printable is untested territory. const SNIFF_TEXT_RATIO = 0.85 -// Charset detection samples the head; below this confidence the guess isn't trusted. 64 KB rather -// than SNIFF_BYTES because this asks a harder question than "is it text": distinguishing among the -// legacy single-byte encodings is a statistical judgement over character frequencies, and jschardet -// gets sharper with more of them. Confidence is jschardet's own 0-1 score, and a miss here is not -// benign — it silently mojibakes the extraction rather than failing — so the bar sits high enough to -// fall back to the latin1 path on a genuinely ambiguous file instead of committing to a coin flip. +// Charset detection samples the head. 64 KB rather than SNIFF_BYTES because this asks a harder +// question than "is it text": telling the legacy single-byte encodings apart is a statistical +// judgement over character frequencies, and jschardet sharpens with more of them. Not currently +// pinned from either side — the suite passes with this at 4 bytes and at 1 GB — so treat it as a +// sampling budget rather than a tuned threshold. const DETECT_SAMPLE_BYTES = 64 * 1024 -const DETECT_MIN_CONFIDENCE = 0.7 ///////////////////////////////////////////////////////////// // TYPES @@ -219,21 +226,16 @@ const resolveCharset = (content: Buffer, hint?: string): string => { if (isUtf8(content) && !content.subarray(0, SNIFF_BYTES).includes(0)) return 'utf-8' // #3. Explicit charset from the Content-Type, if iconv knows it. if (hint && iconv.encodingExists(hint)) return hint - // #4. Statistical detection on the head, gated on confidence. + // #4. Statistical detection on the head, DELIBERATELY UNGATED on jschardet's confidence score. + // Step 2 ruled out utf-8, so these are single-byte legacy bytes: decoding them as utf-8 would + // turn every high byte into an irreversible U+FFFD, and a low-confidence guess still beats + // that. A confidence gate used to sit here and could not ever change the answer — the branch + // below it returned the same `detected.encoding` the gate had just rejected, so every value + // of the threshold produced identical output. Re-adding one is not a tightening: it means + // choosing latin1 over a plausible guess, which is a behaviour change and needs its own case. const detected = jschardet.detect(content.subarray(0, DETECT_SAMPLE_BYTES)) - if ( - detected && - detected.encoding && - detected.confidence > DETECT_MIN_CONFIDENCE && - iconv.encodingExists(detected.encoding) - ) { - return detected.encoding - } - // #5. Floor. Step 2 ruled out utf-8, so these are single-byte legacy bytes and decoding as utf-8 - // would turn every high byte into an irreversible U+FFFD. Take jschardet's guess even below - // the gate (a plausible decode beats guaranteed U+FFFD), else latin1 — maybe wrong, but it - // maps every byte, so it's reversible. if (detected?.encoding && iconv.encodingExists(detected.encoding)) return detected.encoding + // #5. Floor. latin1 maps every byte, so it is wrong-but-reversible rather than lossy. return 'latin1' } diff --git a/body.ts b/body.ts index bc225ea..aa8c80a 100644 --- a/body.ts +++ b/body.ts @@ -4,22 +4,26 @@ // Peer module to attachment.ts; neither depends on the other. Both are surfaced together // from the package entry (index.ts) and each also has its own subpath export. -// Three different lookaheads, deliberately three different sizes — each is bounded by what the -// structure it scans can actually span, not by a shared "peek ahead a bit". -// -// An attribution ("On , wrote:") is ONE logical line that a client hard-wrapped. -// At typical 72-80 column wrapping the longest of them — a long display name plus a bracketed -// address plus a spelled-out date — runs to three physical lines and no further, so stitching line i -// with the two after it covers the real cases. Kept tight on purpose: every extra line widens the -// window in which an unrelated following line can supply the "wrote:" that triggers a wrong cut. +// Three line lookaheads. Each is a FLOOR demonstrated by the corpus, not a window: the corpus shows +// what each one must be at least, and none of them has a case that breaks when it is raised. They +// are kept at the demonstrated minimum (plus, for the divider, a margin) on the general principle +// that a wider scan has more chances to match something unrelated — a risk that is argued, not +// measured, so treat the low side as evidence and the high side as judgement. + +// An attribution ("On , wrote:") is ONE logical line the client hard-wrapped, so +// this stitches line i with the next few before testing. 2 is the smallest value that keeps every +// wrapped attribution in the corpus: at 1 one message miscuts, at 0 six do. Raising it to 3, 4, 6, +// 10 or 50 changes no output at all, so the corpus does not say a tighter window is safer. const ATTRIBUTION_WRAP_LINES = 2 // A "_____" divider is ambiguous (signature rule vs. quote header), so it is only believed when a -// real quote signal follows. Clients put at most a blank line and a header line between the two; -// three NON-EMPTY lines is that, with slack for a client that inserts its own label first. +// real quote signal follows within this many NON-EMPTY lines (blanks skipped, not counted). Only +// being non-zero is demonstrated: every value from 1 to 1000 is identical on the corpus, and only 0 +// regresses. Left at 3 — a blank plus a header line, with room for a client that labels the block — +// rather than 1, because an unbounded scan would recurse over the rest of the message. const DIVIDER_LOOKAHEAD_LINES = 3 -// A pasted "From:" header block is corroborated by its siblings — Sent:/Date:, To:, Subject:. Those -// three are what immediately follow From: in every client's paste format, so this spans the block -// without reaching the message body underneath it. +// A pasted "From:" header block is corroborated by its siblings — Sent:/Date:, To:, Subject: — which +// is what this spans. 2 is the demonstrated minimum (at 1 one message miscuts, at 0 three do); 4, 6, +// 20 and 1000 are all identical. Left at 3 to cover the three sibling headers a paste actually has. const HEADER_BLOCK_LINES = 3 // How far past the opener the guards may look for the rest of an attribution. One budget, used by @@ -27,7 +31,10 @@ const HEADER_BLOCK_LINES = 3 // bounds would let a guard match against text the consuming scan never reaches (or vice versa), // which is how a lookahead-corroborated match becomes a cut in the wrong place. 240 covers the // longest real attributions — a spelled-out date, a long display name and a bracketed address — -// while keeping each scan linear in a bounded window rather than the whole line. +// while keeping each scan linear in a bounded window rather than the whole line. Measured floor is +// about 80 (the corpus's longest glued attribution spans ~62 characters between "On" and "wrote:"; +// 40 regresses it, 80 does not), and nothing constrains it from above out to 2000 — so the value is +// roughly 3x the worst observed case, and the headroom is deliberate rather than load-bearing. const ATTRIBUTION_SCAN_CHARS = 240 // Same line, bounded: [^\r\n] so a scan can never cross into the next one. const ATTRIBUTION_SCAN = `[^\\r\\n]{0,${ATTRIBUTION_SCAN_CHARS}}` @@ -163,6 +170,32 @@ const isQuoteSignal = (lines: string[], i: number): boolean => { return false } +// Thresholds for rescuing a signature stranded below a quote. All three were tuned against a corpus +// that is not in this repo; the bounds recorded here are what the 161-message test corpus can still +// demonstrate, so they narrow the safe range rather than re-deriving the original values. + +// Share of the block between the cut and a "--" that must be quoted lines for that "--" to count as +// sitting just after the quote (rather than being some unrelated "--" further down). Genuinely +// two-sided: at 0 a pasted "From:" header block with no quote markers at all qualifies and its +// trailing line is wrongly reattached, and at 0.75 or above the second marker in a +// quote -> "--" -> inner sig -> "--" -> real sig layout stops qualifying, so the inner (quoted) +// signature is kept instead of dropped. 0.5 sits between the two demonstrated failures. +const QUOTED_BLOCK_RATIO = 0.5 +// Non-empty lines a "--" block needs before it counts as the sender's real signature rather than a +// one-line scrap of quoted junk. At 1 the inner signature of that same layout is treated as +// substantial and kept. Nothing in the corpus constrains it from above (3, 4 and 10 are identical), +// so this is a floor, not a window. +const SIG_BODY_LINES = 2 +// Visible characters a trailing HTML block may have and still be reattached as a signature. The only +// threshold here with a demonstrated bound on BOTH sides: a real multi-line sig (name, title, +// company, address, phone, LinkedIn) runs about 110 characters and is dropped below that, while a +// 370-character block is meant to be refused and is reattached once this goes to 500. Counted in +// visible characters, not lines, because HTML splits signatures across many short lines/cells. +const SIG_VISIBLE_CHARS = 400 +// Markup a "more quoting below" guard may span before finding its "wrote:". 50 is enough for every +// corpus case; kept at HTML_MARKER_SPAN's neighbourhood since it is the same kind of scan. +const MORE_QUOTE_SPAN = 200 + // Forwards // Detects ---------- Forwarded message ---------- or begin forwarded message: const RE_FWD_LINE = /^(?:-+\s*forwarded message\s*-+|begin forwarded message:?)\s*$/i @@ -260,14 +293,14 @@ export const extractNewContent = (text: string): string => { if (!/^--\s*$/.test(lines[j])) continue const body = lines.slice(cut + 1, j).filter((l) => l.trim()) const quoted = body.filter((l) => l.startsWith('>')).length - if (body.length && quoted >= body.length * 0.5) markers.push(j) + if (body.length && quoted >= body.length * QUOTED_BLOCK_RATIO) markers.push(j) } let sigIdx = -1 // Earliest marker whose sig body (up to the next marker) is a substantial ≥2-line block. for (const j of markers) { const nextMarker = markers.find((k) => k > j) ?? lines.length const sigBody = lines.slice(j + 1, nextMarker).filter((l) => l.trim()) - if (sigBody.length >= 2) { + if (sigBody.length >= SIG_BODY_LINES) { sigIdx = j break } @@ -290,6 +323,24 @@ export const extractNewContent = (text: string): string => { // HTML VERSION // HTML counterpart to isQuoteSignal. HTML has no lines, so we search for a LIST of // markers and cut at whichever appears earliest. +// Characters of markup a marker may span before reaching what corroborates it — a "wrote:", an +// address, a colon. ONE constant for all three scans below, because they are one idea: allow the +// tags a client puts between an attribution's parts. It is not what keeps a marker inside its own +// block — the negative lookahead on the closing tag does that — which is why the bound can be +// generous. Measured over the corpus, the widest real span is 184 characters, Outlook's inline CSS +// between "From:" and the address, so this sits just above the worst observed case. +// +// Replaces three separately-chosen values (200 / 250 / 80). All three produce identical output on +// every corpus input at any shared value from 200 to 400, and the 80 was the least grounded of them: +// it fires on one HTML input in the whole corpus, with a 9-character match. +const HTML_MARKER_SPAN = 250 +// Stop a scan at the enclosing block's close, so a marker cannot pair with a corroborator in some +// later block. "On ... wrote:" also stops at list tags; the other two never sit inside a list. +const BLOCK_TAGS = 'div|p|td|tr|table|blockquote|body' +const notPast = (tags: string): string => String.raw`(?:(?!<\/(?:${tags}))[\s\S]){0,${HTML_MARKER_SPAN}}?` +const HTML_FOREIGN_VERBS = + '(a [ée]crit|escribi[óo]|ha scritto|escreveu|schrieb|schreef|geschreven|verzond|skrev|napisał|написал|đã viết|yazd[ıi]|a scris)' + const HTML_QUOTE_MARKERS: RegExp[] = [ // Client wrappers — each client marks quotes its own way. /]*gmail_quote/i, // Gmail @@ -300,11 +351,11 @@ const HTML_QUOTE_MARKERS: RegExp[] = [ // Mirrors #4 — "On ... wrote:". Tags can sit between them, so allow 200 chars; // the inner guard stops "On" pairing with a "wrote:" in a different block. - /\bOn\b(?:(?!<\/(?:div|p|td|tr|table|blockquote|body|ul|ol|li))[\s\S]){0,200}?wrote:/i, + new RegExp(String.raw`\bOn\b` + notPast(`${BLOCK_TAGS}|ul|ol|li`) + 'wrote:', 'i'), // Mirrors #5 — "From:" header. Lookbehind keeps "From" a standalone word, then needs an email. - /(?, empty

gaps) so the spliced HTML stays clean. @@ -370,7 +424,7 @@ const reattachTrailingSignature = (html: string, cutAt: number): string | undefi // Guard 3: what's LEFT after trimming noise must be a real, short signature — bounded by VISIBLE // CHARACTER count (not line count: HTML splits sigs into many short lines/table-cells, so a line // cap misfires on whitespace-padded sigs). Empty (tail was pure noise) → bail to a plain cut. - if (sigVis && sigVis.length <= 400) { + if (sigVis && sigVis.length <= SIG_VISIBLE_CHARS) { return html.slice(0, cutAt) + sig // reply + signature, quoted middle + footer dropped } } From 06bfdf4ccd0ce1f3db5f61ac108ff0794df2b4b7 Mon Sep 17 00:00:00 2001 From: Haakam Aujla Date: Fri, 18 Sep 2026 06:07:24 +0000 Subject: [PATCH 2/3] Make the surrogate trim unconditional, dropping CAP_STORAGE_SLACK MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CAP_STORAGE_SLACK existed only so the central trim could notice that a handler had overshot its cap, because the trailing-surrogate check was gated on that overshoot. That made a correctness guarantee depend on every incremental handler agreeing to store two units more than it can ever return. Check it unconditionally instead: a trailing high surrogate is half a character whether we cut there or a handler stopped there, and stripping it is right either way — left in place it serialises as U+FFFD. The docx reader can then store exactly maxOutputChars and the constant goes away. Truncation is still reported, from the handler's own capExceeded rather than inferred from length. Coverage is unchanged, not reduced: 'never splits a surrogate pair when the cap lands mid-character' now exercises the new path. With storage flush against the cap that document ends on a lone high surrogate WITHOUT overshooting, so the old gated check would miss it — verified by running the two changes separately, where zero slack against the old check fails exactly that test. This is a behaviour change, narrowly: a document that legitimately ends in an unpaired high surrogate now has it stripped rather than emitted as U+FFFD. That input is malformed in every format we read, and both answers lose the half — so this is a better failure, but it is a different one, which is why it is split out from the pure-cleanup commit before it. Co-Authored-By: Claude Opus 5 (1M context) --- attachment.ts | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/attachment.ts b/attachment.ts index b3699ad..335260d 100644 --- a/attachment.ts +++ b/attachment.ts @@ -21,15 +21,6 @@ export const MAX_INPUT_BYTES = 10 * 1024 * 1024 // Cutting sets `truncated` — a partial extraction that reads as complete is worse than a missing one. export const MAX_OUTPUT_CHARS = 250_000 -// Extra UTF-16 units an incrementally-building handler keeps PAST its cap, so the central trim can -// do its job. That trim detects overshoot as `text.length > maxOutputChars` and only then checks -// whether the boundary splits a surrogate pair — so a handler that stored exactly maxOutputChars -// would report a still-continuing document as exactly-at-cap, skip the check, and emit a lone -// surrogate half as U+FFFD. One unit is what makes the overshoot visible and is the strict minimum -// (verified: dropping to zero fails the docx surrogate test); the second is slack so an astral -// character landing on the boundary is stored whole rather than as a half the trim must discard. -const CAP_STORAGE_SLACK = 2 - // Blocks a handler concatenates into one extraction — PDF pages, XLSX sheets — are joined by this. // The incremental length accounting has to charge for it BEFORE the join, so both must read it from // here: a separator that disagrees with its own charged width makes the cap off by that difference. @@ -2474,7 +2465,10 @@ const createDocxReader = async (maxOutputChars: number): Promise => let discardDeferredText = false let deferredFreshFrames: Set | undefined let discardTableText = false - const storageLimit = maxOutputChars + CAP_STORAGE_SLACK + // Exactly the cap: nothing is stored past what can be returned. The entry point strips a + // trailing surrogate half unconditionally, so stopping flush against the cap cannot split a + // character, and truncation is reported from capExceeded rather than inferred from overshoot. + const storageLimit = maxOutputChars const appendRaw = (frame: DocxFrame, field: 'value' | 'extra' | 'claimedExtra' | 'allValue', text: string): void => { const value = frame[field] ?? '' @@ -3323,11 +3317,13 @@ export const extractAttachment = async ( // host memory limit (see README). Don't split a surrogate pair at the boundary: a lone half // serializes as U+FFFD. const overCap = output.text.length > maxOutputChars - const capEnd = - overCap && output.text.charCodeAt(maxOutputChars - 1) >= 0xd800 && output.text.charCodeAt(maxOutputChars - 1) <= 0xdbff - ? maxOutputChars - 1 - : maxOutputChars - const text = overCap ? output.text.slice(0, capEnd) : output.text + const capped = overCap ? output.text.slice(0, maxOutputChars) : output.text + // A trailing high surrogate is half a character however it got there — our cut, or a handler + // that stopped exactly on the cap. Checked unconditionally rather than only when we cut, + // because the alternative is requiring every incremental handler to store past its own cap + // purely so this check can notice, which is what the CAP_STORAGE_SLACK constant used to buy. + const lastUnit = capped.charCodeAt(capped.length - 1) + const text = lastUnit >= 0xd800 && lastUnit <= 0xdbff ? capped.slice(0, -1) : capped // Decided on the FINAL text: a tight enough cap slices a non-empty extraction to '', and the // contract is that `extraction` is omitted rather than ever being ''. `||`, not `??`, because // the handler's call can only ADD emptiness — pdf computes `empty` from its pre-cap page join From d9f113cda2e23514bd41c34b53303183cd91c97f Mon Sep 17 00:00:00 2001 From: Haakam Aujla Date: Sat, 19 Sep 2026 22:15:37 +0000 Subject: [PATCH 3/3] Restore the foreign-verb scan's narrower window MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review caught a real regression in the previous commit: unifying the three HTML marker windows at 250 widened the foreign-verb scan from 80, and ordinary French prose then matches. "Paul a écrit une proposition détaillée ... Voici mon avis :" is a plain reply whose own later colon corroborates the verb, so everything below the greeting was cut away as quoted history. The evidence behind that unification was too weak to carry it, and the shape of the mistake is worth recording. The marker fires on ONE of 49 HTML inputs in the corpus, with a 9-character match — so the corpus had no power to detect the change, and "0 outputs changed" measured its own blind spot rather than safety. I stated that caveat on the method and then did not apply it to this decision. The three windows are not one number, because how far a scan may wander depends on how much evidence it is looking for. "On ... wrote:" needs a literal "wrote:" and the From: scan needs a full email address; ordinary prose supplies neither, so they can afford a wide window. The foreign-verb scan needs only a COLON, and its verbs are ordinary words — the window IS the guard there, and the closing-tag lookahead does not help, because the trailing colon is inside the same
. Restored as three named spans sharing the notPast helper, so the readability of the refactor survives without the behaviour change. All twelve compiled markers are now byte-identical to main, verified by comparing source and flags. Regression test pinned with a ~125-character gap — inside a 250-char window and outside the 80-char one — and confirmed to fail when the span is set back to 250. Co-Authored-By: Claude Opus 5 (1M context) --- body.ts | 36 ++++++++++++++++++++++++------------ tests/body.test.ts | 15 +++++++++++++++ 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/body.ts b/body.ts index aa8c80a..8d48a8f 100644 --- a/body.ts +++ b/body.ts @@ -323,21 +323,33 @@ export const extractNewContent = (text: string): string => { // HTML VERSION // HTML counterpart to isQuoteSignal. HTML has no lines, so we search for a LIST of // markers and cut at whichever appears earliest. -// Characters of markup a marker may span before reaching what corroborates it — a "wrote:", an -// address, a colon. ONE constant for all three scans below, because they are one idea: allow the -// tags a client puts between an attribution's parts. It is not what keeps a marker inside its own -// block — the negative lookahead on the closing tag does that — which is why the bound can be -// generous. Measured over the corpus, the widest real span is 184 characters, Outlook's inline CSS -// between "From:" and the address, so this sits just above the worst observed case. +// Characters of markup each marker may span before reaching the thing that corroborates it. These +// are NOT one number. They were briefly unified at 250 on the grounds that no corpus output changed; +// that was wrong, and the case below is why — how far a scan may wander depends on how much evidence +// the thing it is looking for actually carries. // -// Replaces three separately-chosen values (200 / 250 / 80). All three produce identical output on -// every corpus input at any shared value from 200 to 400, and the 80 was the least grounded of them: -// it fires on one HTML input in the whole corpus, with a 9-character match. +// "On ... wrote:" and "From:
" are strongly anchored: ordinary prose does not contain a +// literal "wrote:", and the From: scan has to reach a full email address. They can afford a wide +// window, and need one — the widest real span in the corpus is 184 characters, Outlook's inline CSS +// sitting between "From:" and the address. const HTML_MARKER_SPAN = 250 +// "On ... wrote:" kept its own 200 rather than being rounded up to the constant above: the corpus's +// longest real match is 90 characters, so 200 is already ~2x the worst case and widening it buys +// nothing measurable while giving a stray "wrote:" more room to pair with an unrelated "On". +const HTML_ON_WROTE_SPAN = 200 +// The foreign-verb scan is the weak one: it only needs a COLON, and its verbs are ordinary words in +// ordinary sentences. "Paul a écrit une proposition détaillée ... Voici mon avis :" is a plain French +// reply, and at a 250-character window that sentence's own later colon corroborates the verb and the +// whole message below it is cut away. 80 keeps the scan inside the attribution-shaped neighbourhood +// where a real "X a écrit :" puts its colon. Do not raise this to match the constant above; if it +// needs to reach further, the fix is more evidence (a date, an address), not a longer leash. +const HTML_FOREIGN_VERB_SPAN = 80 // Stop a scan at the enclosing block's close, so a marker cannot pair with a corroborator in some // later block. "On ... wrote:" also stops at list tags; the other two never sit inside a list. +// NOTE the block guard does not save the foreign-verb case above: that colon is in the SAME
. const BLOCK_TAGS = 'div|p|td|tr|table|blockquote|body' -const notPast = (tags: string): string => String.raw`(?:(?!<\/(?:${tags}))[\s\S]){0,${HTML_MARKER_SPAN}}?` +const notPast = (tags: string, span = HTML_MARKER_SPAN): string => + String.raw`(?:(?!<\/(?:${tags}))[\s\S]){0,${span}}?` const HTML_FOREIGN_VERBS = '(a [ée]crit|escribi[óo]|ha scritto|escreveu|schrieb|schreef|geschreven|verzond|skrev|napisał|написал|đã viết|yazd[ıi]|a scris)' @@ -351,11 +363,11 @@ const HTML_QUOTE_MARKERS: RegExp[] = [ // Mirrors #4 — "On ... wrote:". Tags can sit between them, so allow 200 chars; // the inner guard stops "On" pairing with a "wrote:" in a different block. - new RegExp(String.raw`\bOn\b` + notPast(`${BLOCK_TAGS}|ul|ol|li`) + 'wrote:', 'i'), + new RegExp(String.raw`\bOn\b` + notPast(`${BLOCK_TAGS}|ul|ol|li`, HTML_ON_WROTE_SPAN) + 'wrote:', 'i'), // Mirrors #5 — "From:" header. Lookbehind keeps "From" a standalone word, then needs an email. new RegExp(String.raw`(? { expect(exHtml(html)).toBe('
Merci, bien noté.
') }) + // The mirror of the test above, and the reason the foreign-verb scan has a TIGHTER window than + // the other HTML markers rather than sharing theirs. That scan's verbs are ordinary words and its + // only corroborator is a colon, so the window is the whole guard: widen it and an unremarkable + // French sentence reaches its own later colon and the reply below is cut away as quoted history. + // Pinned with a ~125-character gap — inside a 250-char window, outside the 80-char one — and in + // one
, so the closing-tag lookahead cannot save it either. + it('keeps French prose where "a écrit" is a verb and a colon follows much later', () => { + const html = + '
Bonjour Marie,
' + + '
Paul a écrit une proposition détaillée pour le nouveau projet et je pense ' + + 'que nous devrions en discuter demain matin. Voici mon avis : elle est solide.
' + + '
Merci !
' + expect(exHtml(html)).toBe(html) + }) + // #7 (Chinese): 写道 ("wrote") verb-last ending in a full-width colon. it('cuts at a Chinese "写道：" attribution in HTML', () => { const html =