diff --git a/openspec/changes/add-read-file-fingerprint-ordinal/proposal.md b/openspec/changes/add-read-file-fingerprint-ordinal/proposal.md new file mode 100644 index 00000000..b0bb95e4 --- /dev/null +++ b/openspec/changes/add-read-file-fingerprint-ordinal/proposal.md @@ -0,0 +1,57 @@ +# Change: Add opt-in fingerprint duplicate-disambiguation metadata to read_file + +## Why + +`read_file(format="json", include_fingerprint=true)` already gives consumers a portable +`content_fingerprint` (`sha256:nfkc:<32hex>`) that is stable across reads, machines, and +re-uploads. The remaining gap is **duplicate disambiguation**: when the same normalized +paragraph text appears multiple times in one document, every occurrence shares one +fingerprint, so a downstream consumer cannot reference "the second occurrence of WHEREAS …" +without computing its own document-order ordinal. + +Today every consumer that hits this (e.g. legal-context's manual-DOCX ingest / citation +pipeline) has to re-implement the same four steps: request fingerprints, group paragraphs by +fingerprint, assign document-order ordinals per group, and build a composite key. That logic +is small but easy to drift on across consumers. (Issue #205.) + +## What Changes + +### safe-docx (MCP) + +- MODIFIED: `read_file` tool (`tools/read_file.ts`) — new opt-in + `include_fingerprint_ordinal: boolean` (default `false`). When it is `true` **and** + `include_fingerprint=true` **and** `format="json"` for a DOCX session, each paragraph node + gains three additional fields: + - `content_fingerprint_ordinal`: 1-based position of the paragraph among all paragraphs in + the document sharing its `content_fingerprint`, in document order. + - `content_fingerprint_count_in_document`: total number of paragraphs in the document + sharing that fingerprint (document-wide, not windowed to the returned slice). + - `portable_paragraph_ref`: the convenience composite `"#"`. + - **Document-wide**: ordinals and counts are computed over the full document in document + order, so a paginated / `node_ids`-filtered read still reports stable ordinals and the + full document count. + - **No effect without `include_fingerprint`**: if `include_fingerprint_ordinal=true` is + passed without `include_fingerprint=true`, no ordinal fields are emitted (the + disambiguator sits on top of the existing fingerprint surface). + - **Read-only disambiguator**: ordinals are NOT edit anchors. Reordering duplicate + paragraphs may change ordinals. Edit tools continue to accept only `_bk_*` IDs. +- MODIFIED: `tool_catalog.ts` — `read_file` input schema gains `include_fingerprint_ordinal`. + +## Impact + +- Affected specs: `mcp-server` (read_file gains opt-in fingerprint ordinal disambiguation). +- Purely additive. Default off; existing consumers see byte-identical output. `id` (`_bk_*`), + edit-anchor semantics, and the `content_fingerprint` algorithm are all unchanged. +- JSON-only: TOON/simple output is unchanged regardless of the flag (same contract as + `include_fingerprint`). Google Docs and ODT sessions ignore the flag. +- Affected code: `packages/docx-mcp/src/tools/read_file.ts`, + `packages/docx-mcp/src/tool_catalog.ts`. Regenerated: + `packages/docx-mcp/docs/tool-reference.generated.md`. + +## Out of scope + +- Replacing `_bk_*` with content-addressable `_p_*` IDs. +- Making portable references valid edit anchors. +- Changing TOON / simple / Google Docs output. +- Any new normalization algorithm or shorter hash format. +- Footnote / endnote / comment fingerprints. diff --git a/openspec/changes/add-read-file-fingerprint-ordinal/specs/mcp-server/spec.md b/openspec/changes/add-read-file-fingerprint-ordinal/specs/mcp-server/spec.md new file mode 100644 index 00000000..cbe400e6 --- /dev/null +++ b/openspec/changes/add-read-file-fingerprint-ordinal/specs/mcp-server/spec.md @@ -0,0 +1,100 @@ +# mcp-server delta — opt-in fingerprint duplicate-disambiguation metadata on read_file + +## ADDED Requirements + +### Requirement: Optional Fingerprint Ordinal Disambiguation on read_file JSON + +The `read_file` tool SHALL accept an optional `include_fingerprint_ordinal` boolean parameter +(default `false`). When `include_fingerprint_ordinal=true` AND `include_fingerprint=true` AND +`format="json"` for a DOCX session, the server SHALL emit three additional fields on each +paragraph node alongside `content_fingerprint`: + +- `content_fingerprint_ordinal`: a 1-based integer giving the paragraph's position, in + document order, among all paragraphs in the document that share its `content_fingerprint`. +- `content_fingerprint_count_in_document`: an integer giving the total number of paragraphs in + the document that share that `content_fingerprint`. +- `portable_paragraph_ref`: the convenience composite string + `"#"`. + +Ordinals and counts SHALL be computed over the **entire document** in document order, not over +the returned (paginated, `offset`/`limit`, or `node_ids`-filtered) slice. A paragraph in a +returned slice SHALL report the same ordinal and count it would report in a full read of the +document. + +For a paragraph whose `content_fingerprint` is unique in the document, the ordinal SHALL be `1` +and the count SHALL be `1`. + +The ordinal is a read-only disambiguator, NOT an edit anchor. Reordering duplicate paragraphs +MAY change ordinals. Edit tools SHALL continue to accept ONLY `_bk_*` identifiers as anchors, +and the paragraph `id` SHALL remain unchanged. The `content_fingerprint` algorithm SHALL remain +unchanged (NFKC normalization, Cf/invisible stripping, whitespace collapse + trim, +`sha256:nfkc:<32hex>`). + +The flag SHALL have no effect unless `include_fingerprint=true` is also set: when +`include_fingerprint_ordinal=true` is passed without `include_fingerprint=true`, no ordinal +fields SHALL be emitted. + +When `include_fingerprint_ordinal=true` is passed with `format="toon"` or `format="simple"`, +the flag SHALL have no effect (TOON and simple outputs are unchanged). When passed with a +Google Docs session (`google_doc_id`), the server SHALL silently ignore the flag. + +#### Scenario: opt-in ordinal adds disambiguation fields on JSON output +- **GIVEN** a DOCX session +- **WHEN** `read_file` is called with `format="json"`, `include_fingerprint=true`, and `include_fingerprint_ordinal=true` +- **THEN** each paragraph object SHALL include integer `content_fingerprint_ordinal` and `content_fingerprint_count_in_document` fields +- **AND** a `portable_paragraph_ref` string of the form `#` + +#### Scenario: unique paragraph fingerprint reports ordinal 1 and count 1 +- **GIVEN** a DOCX session whose paragraphs all have distinct normalized text +- **WHEN** `read_file` is called with `format="json"`, `include_fingerprint=true`, and `include_fingerprint_ordinal=true` +- **THEN** every paragraph SHALL report `content_fingerprint_ordinal` of `1` +- **AND** `content_fingerprint_count_in_document` of `1` + +#### Scenario: duplicate normalized text receives deterministic document-order ordinals +- **GIVEN** a DOCX session with the same normalized paragraph text appearing three times +- **WHEN** `read_file` is called with `format="json"`, `include_fingerprint=true`, and `include_fingerprint_ordinal=true` +- **THEN** the three duplicate paragraphs SHALL receive `content_fingerprint_ordinal` values `1`, `2`, `3` in document order +- **AND** each SHALL report `content_fingerprint_count_in_document` of `3` + +#### Scenario: whitespace-only variants share fingerprint and get distinct ordinals +- **GIVEN** a DOCX session with two paragraphs whose visible text differs only by collapsible whitespace +- **WHEN** `read_file` is called with `format="json"`, `include_fingerprint=true`, and `include_fingerprint_ordinal=true` +- **THEN** the two paragraphs SHALL share the same `content_fingerprint` +- **AND** SHALL receive distinct `content_fingerprint_ordinal` values `1` and `2` + +#### Scenario: ordinal fields require include_fingerprint +- **GIVEN** a DOCX session +- **WHEN** `read_file` is called with `format="json"` and `include_fingerprint_ordinal=true` but without `include_fingerprint` +- **THEN** paragraph objects SHALL NOT contain `content_fingerprint_ordinal`, `content_fingerprint_count_in_document`, or `portable_paragraph_ref` + +#### Scenario: portable_paragraph_ref composes fingerprint and ordinal +- **GIVEN** a DOCX session +- **WHEN** `read_file` is called with `format="json"`, `include_fingerprint=true`, and `include_fingerprint_ordinal=true` +- **THEN** each `portable_paragraph_ref` SHALL equal that node's `content_fingerprint` followed by `#` and its `content_fingerprint_ordinal` + +#### Scenario: counts are document-wide across paginated windows +- **GIVEN** a DOCX session with a duplicated paragraph appearing three times across the document +- **WHEN** `read_file` is called with a `limit`/`node_ids` window that returns only some of the duplicates plus `include_fingerprint=true` and `include_fingerprint_ordinal=true` +- **THEN** the returned duplicate paragraphs SHALL report `content_fingerprint_count_in_document` of `3` +- **AND** their `content_fingerprint_ordinal` values SHALL match their document-order positions + +#### Scenario: default JSON output omits ordinal fields +- **GIVEN** a DOCX session +- **WHEN** `read_file` is called with `format="json"` and `include_fingerprint=true` but no `include_fingerprint_ordinal` +- **THEN** paragraph objects SHALL contain `content_fingerprint` but NOT `content_fingerprint_ordinal`, `content_fingerprint_count_in_document`, or `portable_paragraph_ref` + +#### Scenario: TOON format ignores include_fingerprint_ordinal +- **GIVEN** a DOCX session +- **WHEN** `read_file` is called with `format="toon"`, `include_fingerprint=true`, and `include_fingerprint_ordinal=true` +- **THEN** the TOON output SHALL be identical to the output produced without the flags + +#### Scenario: simple format ignores include_fingerprint_ordinal +- **GIVEN** a DOCX session +- **WHEN** `read_file` is called with `format="simple"`, `include_fingerprint=true`, and `include_fingerprint_ordinal=true` +- **THEN** the simple output SHALL be identical to the output produced without the flags + +#### Scenario: Google Docs ignores include_fingerprint_ordinal +- **GIVEN** a Google Docs session +- **WHEN** `read_file` is dispatched with `google_doc_id`, `format="json"`, `include_fingerprint=true`, and `include_fingerprint_ordinal=true` +- **THEN** the call SHALL succeed +- **AND** gdocs nodes SHALL NOT contain `content_fingerprint_ordinal`, `content_fingerprint_count_in_document`, or `portable_paragraph_ref` diff --git a/openspec/changes/add-read-file-fingerprint-ordinal/tasks.md b/openspec/changes/add-read-file-fingerprint-ordinal/tasks.md new file mode 100644 index 00000000..527b35c0 --- /dev/null +++ b/openspec/changes/add-read-file-fingerprint-ordinal/tasks.md @@ -0,0 +1,23 @@ +# Tasks: Add opt-in fingerprint duplicate-disambiguation metadata to read_file + +## 1. docx-mcp read_file tool +- [x] 1.1 `read_file.ts`: add `include_fingerprint_ordinal` param; when it and + `include_fingerprint` are set with `format="json"`, compute document-wide, + document-order ordinals and counts per `content_fingerprint`, and attach + `content_fingerprint_ordinal`, `content_fingerprint_count_in_document`, and + `portable_paragraph_ref` to each JSON node. No effect without `include_fingerprint`. +- [x] 1.2 `tool_catalog.ts`: expose `include_fingerprint_ordinal` on the `read_file` input + schema. +- [x] 1.3 Regenerate `docs/tool-reference.generated.md`. + +## 2. Tests (mapped to spec scenarios) +- [x] 2.1 `read_file_fingerprint_ordinal.test.ts` in docx-mcp with + `TEST_FEATURE='add-read-file-fingerprint-ordinal'`: opt-in fields, unique paragraph + ordinal/count, duplicate document-order ordinals, whitespace-variant grouping, + ordinal requires include_fingerprint, portable_paragraph_ref composition, + document-wide counts across windowed reads, TOON ignores the flag, default JSON omits + the fields, and Google Docs ignores the flag. + +## 3. Validation +- [x] 3.1 `openspec validate add-read-file-fingerprint-ordinal --strict`. +- [x] 3.2 Build, lint, targeted tests, spec-coverage, tool-docs check. diff --git a/packages/docx-mcp/docs/tool-reference.generated.md b/packages/docx-mcp/docs/tool-reference.generated.md index 92cee775..e98d871a 100644 --- a/packages/docx-mcp/docs/tool-reference.generated.md +++ b/packages/docx-mcp/docs/tool-reference.generated.md @@ -23,6 +23,7 @@ Read document content (DOCX, ODT, or Google Doc). Output is token-limited (~14k | `comment_rendering` | `enum("none", "paragraph_notes", "endnotes", "inline_markers")` | no | How to render comments in read_file output. Use "paragraph_notes" (default) for paragraph-local comment threads, "inline_markers" to add `[cm-start:N]`/`[cm-end:N]` milestones in TOON output (combined with the thread blocks), "endnotes" to collect threaded comments into a trailing #COMMENTS block in TOON output, or "none" for the legacy output with no comment rendering. | | `show_formatting` | `boolean` | no | When true (default), shows inline formatting tags (, , , , ). When false, emits plain text with no inline tags. | | `include_fingerprint` | `boolean` | no | When true and format="json", include a portable content_fingerprint ("sha256:nfkc:<32hex>") on each paragraph. Read-only metadata derived from the paragraph's normalized visible text; NOT an edit anchor. Edit tools accept only `_bk_*` IDs. No effect on TOON/simple output. Ignored for Google Docs and ODT. | +| `include_fingerprint_ordinal` | `boolean` | no | When true together with include_fingerprint and format="json", add duplicate-disambiguation metadata to each paragraph: `content_fingerprint_ordinal` (1-based document-order position among paragraphs sharing the same content_fingerprint), `content_fingerprint_count_in_document` (total paragraphs sharing it, document-wide even under pagination), and `portable_paragraph_ref` ("#"). Read-only disambiguator, NOT an edit anchor; reordering duplicates may change ordinals. No effect without include_fingerprint, and no effect on TOON/simple output. Ignored for Google Docs and ODT. Default: false. | | `include_footnotes` | `boolean` | no | When true and format="json", attach a `footnotes` array ({id, display_number, text}) to each paragraph node for the footnotes anchored to it. Windowed to the returned slice (a paginated walk returns each footnote exactly once) and counted toward the read token budget. Footnotes with an empty body or no anchored paragraph are excluded — use get_footnotes for the authoritative full enumeration. No effect on TOON/simple output. Ignored for Google Docs and ODT. Default: false. | ## `grep` diff --git a/packages/docx-mcp/src/testing/SAFE_DOCX_OPENSPEC_TRACEABILITY.md b/packages/docx-mcp/src/testing/SAFE_DOCX_OPENSPEC_TRACEABILITY.md index c4763d29..071d0f16 100644 --- a/packages/docx-mcp/src/testing/SAFE_DOCX_OPENSPEC_TRACEABILITY.md +++ b/packages/docx-mcp/src/testing/SAFE_DOCX_OPENSPEC_TRACEABILITY.md @@ -219,6 +219,21 @@ This matrix maps OpenSpec `#### Scenario:` entries to Allure story mappings extr | Two-file `.odt` compare reports inline granularity and meaningful modifications | covered | `src/tools/odf/odf_compare_inline.test.ts` | | | Whole-paragraph-only diffs still report zero modifications | covered | `src/tools/odf/odf_compare_inline.test.ts` | | +## Change: `add-read-file-fingerprint-ordinal` + +| Scenario | Status | Allure Test Files | Notes | +|---|---|---|---| +| Google Docs ignores include_fingerprint_ordinal | covered | `src/tools/read_file_fingerprint_ordinal.test.ts` | | +| TOON format ignores include_fingerprint_ordinal | covered | `src/tools/read_file_fingerprint_ordinal.test.ts` | | +| counts are document-wide across paginated windows | covered | `src/tools/read_file_fingerprint_ordinal.test.ts` | | +| default JSON output omits ordinal fields | covered | `src/tools/read_file_fingerprint_ordinal.test.ts` | | +| duplicate normalized text receives deterministic document-order ordinals | covered | `src/tools/read_file_fingerprint_ordinal.test.ts` | | +| opt-in ordinal adds disambiguation fields on JSON output | covered | `src/tools/read_file_fingerprint_ordinal.test.ts` | | +| ordinal fields require include_fingerprint | covered | `src/tools/read_file_fingerprint_ordinal.test.ts` | | +| portable_paragraph_ref composes fingerprint and ordinal | covered | `src/tools/read_file_fingerprint_ordinal.test.ts` | | +| unique paragraph fingerprint reports ordinal 1 and count 1 | covered | `src/tools/read_file_fingerprint_ordinal.test.ts` | | +| whitespace-only variants share fingerprint and get distinct ordinals | covered | `src/tools/read_file_fingerprint_ordinal.test.ts` | | + ## Change: `add-read-file-inline-footnotes` | Scenario | Status | Allure Test Files | Notes | diff --git a/packages/docx-mcp/src/tool_catalog.ts b/packages/docx-mcp/src/tool_catalog.ts index 4d9eb187..699fa4bc 100644 --- a/packages/docx-mcp/src/tool_catalog.ts +++ b/packages/docx-mcp/src/tool_catalog.ts @@ -56,6 +56,12 @@ export const SAFE_DOCX_TOOL_CATALOG = [ .describe( 'When true and format="json", include a portable content_fingerprint ("sha256:nfkc:<32hex>") on each paragraph. Read-only metadata derived from the paragraph\'s normalized visible text; NOT an edit anchor. Edit tools accept only `_bk_*` IDs. No effect on TOON/simple output. Ignored for Google Docs and ODT.', ), + include_fingerprint_ordinal: z + .boolean() + .optional() + .describe( + 'When true together with include_fingerprint and format="json", add duplicate-disambiguation metadata to each paragraph: `content_fingerprint_ordinal` (1-based document-order position among paragraphs sharing the same content_fingerprint), `content_fingerprint_count_in_document` (total paragraphs sharing it, document-wide even under pagination), and `portable_paragraph_ref` ("#"). Read-only disambiguator, NOT an edit anchor; reordering duplicates may change ordinals. No effect without include_fingerprint, and no effect on TOON/simple output. Ignored for Google Docs and ODT. Default: false.', + ), include_footnotes: z .boolean() .optional() diff --git a/packages/docx-mcp/src/tools/read_file.ts b/packages/docx-mcp/src/tools/read_file.ts index 38cac637..b644b9d4 100644 --- a/packages/docx-mcp/src/tools/read_file.ts +++ b/packages/docx-mcp/src/tools/read_file.ts @@ -296,6 +296,7 @@ export async function readFile( show_formatting?: boolean; comment_rendering?: string; include_fingerprint?: boolean; + include_fingerprint_ordinal?: boolean; include_footnotes?: boolean; }, ): Promise { @@ -408,11 +409,55 @@ export async function readFile( // which has list labels stripped and footnote markers appended above. let jsonNodes: readonly Record[] = enriched; if (params.include_fingerprint && format === 'json') { + // Opt-in duplicate-disambiguation metadata (#205). When + // include_fingerprint_ordinal is also set, compute document-order ordinals + // and counts per fingerprint over the FULL document (not the returned + // slice) so a paginated / node_ids-filtered read still reports stable, + // document-wide ordinals and counts. The ordinal is a read-only + // disambiguator, never an edit anchor. + const ordinalByNodeId = new Map(); + // Cache fingerprints computed during the ordinal pass so the per-node + // enrichment below reuses them instead of recomputing the same + // computeContentFingerprint(getParagraphText(...)) for every windowed + // node. Both sites key off the same paragraph element, so the value is + // identical — this only avoids the double compute (#205). + const fingerprintByNodeId = new Map(); + if (params.include_fingerprint_ordinal) { + const groupCounts = new Map(); + for (const node of nodes) { + const paragraphEl = paragraphElementsById.get(node.id); + if (!paragraphEl) continue; + const fp = computeContentFingerprint(getParagraphText(paragraphEl)); + fingerprintByNodeId.set(node.id, fp); + groupCounts.set(fp, (groupCounts.get(fp) ?? 0) + 1); + } + const runningOrdinal = new Map(); + for (const node of nodes) { + const fp = fingerprintByNodeId.get(node.id); + if (fp == null) continue; + const ordinal = (runningOrdinal.get(fp) ?? 0) + 1; + runningOrdinal.set(fp, ordinal); + ordinalByNodeId.set(node.id, { ordinal, count: groupCounts.get(fp)! }); + } + } + jsonNodes = enriched.map((node) => { const paragraphEl = paragraphElementsById.get(node.id); if (!paragraphEl) return node; - const fingerprint = computeContentFingerprint(getParagraphText(paragraphEl)); - return { ...node, content_fingerprint: fingerprint }; + const fingerprint = + fingerprintByNodeId.get(node.id) ?? + computeContentFingerprint(getParagraphText(paragraphEl)); + const withFingerprint: Record = { + ...node, + content_fingerprint: fingerprint, + }; + const ordinalInfo = ordinalByNodeId.get(node.id); + if (ordinalInfo) { + withFingerprint.content_fingerprint_ordinal = ordinalInfo.ordinal; + withFingerprint.content_fingerprint_count_in_document = ordinalInfo.count; + withFingerprint.portable_paragraph_ref = `${fingerprint}#${ordinalInfo.ordinal}`; + } + return withFingerprint; }); } diff --git a/packages/docx-mcp/src/tools/read_file_fingerprint_ordinal.test.ts b/packages/docx-mcp/src/tools/read_file_fingerprint_ordinal.test.ts new file mode 100644 index 00000000..d6c5b81e --- /dev/null +++ b/packages/docx-mcp/src/tools/read_file_fingerprint_ordinal.test.ts @@ -0,0 +1,484 @@ +import { describe, expect, vi } from 'vitest'; +import { testAllure, type AllureBddContext } from '../testing/allure-test.js'; +import { + openSession, + assertSuccess, + registerCleanup, + createTestSessionManager, +} from '../testing/session-test-utils.js'; +import { readFile } from './read_file.js'; +import { dispatchToolCall } from '../server.js'; +import { SessionManager } from '../session/manager.js'; + +const TEST_FEATURE = 'add-read-file-fingerprint-ordinal'; + +type JsonNode = Record; + +function parseNodes(content: unknown): JsonNode[] { + return JSON.parse(String(content)) as JsonNode[]; +} + +describe('add-read-file-fingerprint-ordinal — Optional Fingerprint Ordinal Disambiguation on read_file JSON', () => { + const test = testAllure.epic('Document Reading').withLabels({ feature: TEST_FEATURE }); + + registerCleanup(); + + test.openspec('opt-in ordinal adds disambiguation fields on JSON output')( + 'opt-in ordinal adds disambiguation fields on JSON output', + async ({ given, when, then }: AllureBddContext) => { + const mgr = createTestSessionManager(); + const { filePath } = await given('a DOCX session', () => + openSession(['First paragraph.', 'Second paragraph.'], { mgr }), + ); + + const nodes = await when( + 'read_file is called with include_fingerprint and include_fingerprint_ordinal', + async () => { + const result = await readFile(mgr, { + file_path: filePath, + format: 'json', + include_fingerprint: true, + include_fingerprint_ordinal: true, + limit: 100, + }); + assertSuccess(result, 'read'); + return parseNodes(result.content); + }, + ); + + await then( + 'each node carries integer content_fingerprint_ordinal/count and a portable_paragraph_ref', + async () => { + expect(nodes.length).toBeGreaterThan(0); + for (const node of nodes) { + expect(Number.isInteger(node.content_fingerprint_ordinal)).toBe(true); + expect(Number.isInteger(node.content_fingerprint_count_in_document)).toBe(true); + expect(node.portable_paragraph_ref).toBe( + `${node.content_fingerprint}#${node.content_fingerprint_ordinal}`, + ); + } + }, + ); + }, + ); + + test.openspec('unique paragraph fingerprint reports ordinal 1 and count 1')( + 'unique paragraph fingerprint reports ordinal 1 and count 1', + async ({ given, when, then }: AllureBddContext) => { + const mgr = createTestSessionManager(); + const { filePath } = await given('a DOCX session with all-distinct paragraphs', () => + openSession(['Alpha clause.', 'Beta clause.', 'Gamma clause.'], { mgr }), + ); + + const nodes = await when('reading with ordinal disambiguation enabled', async () => { + const result = await readFile(mgr, { + file_path: filePath, + format: 'json', + include_fingerprint: true, + include_fingerprint_ordinal: true, + limit: 100, + }); + assertSuccess(result, 'read'); + return parseNodes(result.content); + }); + + await then('every paragraph reports ordinal 1 and count 1', async () => { + for (const node of nodes) { + expect(node.content_fingerprint_ordinal).toBe(1); + expect(node.content_fingerprint_count_in_document).toBe(1); + } + }); + }, + ); + + test.openspec('duplicate normalized text receives deterministic document-order ordinals')( + 'duplicate normalized text receives deterministic document-order ordinals', + async ({ given, when, then }: AllureBddContext) => { + const dup = 'WHEREAS the parties agree.'; + const mgr = createTestSessionManager(); + const { filePath } = await given('a DOCX session with the same text three times', () => + openSession([dup, 'Filler one.', dup, 'Filler two.', dup], { mgr }), + ); + + const nodes = await when('reading with ordinal disambiguation enabled', async () => { + const result = await readFile(mgr, { + file_path: filePath, + format: 'json', + include_fingerprint: true, + include_fingerprint_ordinal: true, + limit: 100, + }); + assertSuccess(result, 'read'); + return parseNodes(result.content); + }); + + await then('the duplicates get ordinals 1,2,3 in document order with count 3', async () => { + const dupFp = nodes[0]!.content_fingerprint; + const dupNodes = nodes.filter((n) => n.content_fingerprint === dupFp); + expect(dupNodes.map((n) => n.content_fingerprint_ordinal)).toEqual([1, 2, 3]); + for (const node of dupNodes) { + expect(node.content_fingerprint_count_in_document).toBe(3); + } + }); + }, + ); + + test.openspec('whitespace-only variants share fingerprint and get distinct ordinals')( + 'whitespace-only variants share fingerprint and get distinct ordinals', + async ({ given, when, then }: AllureBddContext) => { + const mgr = createTestSessionManager(); + // Same words; the second copy has a collapsible double space. The + // fingerprint algorithm collapses runs of whitespace, so both hash equal. + const { filePath } = await given('a DOCX with two whitespace-variant paragraphs', () => + openSession(['Reserved section text.', 'Reserved section text.'], { mgr }), + ); + + const nodes = await when('reading with ordinal disambiguation enabled', async () => { + const result = await readFile(mgr, { + file_path: filePath, + format: 'json', + include_fingerprint: true, + include_fingerprint_ordinal: true, + limit: 100, + }); + assertSuccess(result, 'read'); + return parseNodes(result.content); + }); + + await then('they share one fingerprint and receive ordinals 1 and 2', async () => { + expect(nodes[0]!.content_fingerprint).toBe(nodes[1]!.content_fingerprint); + expect(nodes[0]!.content_fingerprint_ordinal).toBe(1); + expect(nodes[1]!.content_fingerprint_ordinal).toBe(2); + expect(nodes[0]!.content_fingerprint_count_in_document).toBe(2); + }); + }, + ); + + test.openspec('ordinal fields require include_fingerprint')( + 'ordinal fields require include_fingerprint', + async ({ given, when, then }: AllureBddContext) => { + const mgr = createTestSessionManager(); + const { filePath } = await given('a DOCX session', () => + openSession(['Only paragraph.'], { mgr }), + ); + + const nodes = await when( + 'reading with include_fingerprint_ordinal but no include_fingerprint', + async () => { + const result = await readFile(mgr, { + file_path: filePath, + format: 'json', + include_fingerprint_ordinal: true, + limit: 100, + }); + assertSuccess(result, 'read'); + return parseNodes(result.content); + }, + ); + + await then('no ordinal disambiguation fields are emitted', async () => { + for (const node of nodes) { + expect(node.content_fingerprint_ordinal).toBeUndefined(); + expect(node.content_fingerprint_count_in_document).toBeUndefined(); + expect(node.portable_paragraph_ref).toBeUndefined(); + } + }); + }, + ); + + test.openspec('portable_paragraph_ref composes fingerprint and ordinal')( + 'portable_paragraph_ref composes fingerprint and ordinal', + async ({ given, when, then }: AllureBddContext) => { + const dup = 'Identical recital text.'; + const mgr = createTestSessionManager(); + const { filePath } = await given('a DOCX session with a duplicated paragraph', () => + openSession([dup, dup], { mgr }), + ); + + const nodes = await when('reading with ordinal disambiguation enabled', async () => { + const result = await readFile(mgr, { + file_path: filePath, + format: 'json', + include_fingerprint: true, + include_fingerprint_ordinal: true, + limit: 100, + }); + assertSuccess(result, 'read'); + return parseNodes(result.content); + }); + + await then('portable_paragraph_ref equals "#"', async () => { + expect(nodes[0]!.portable_paragraph_ref).toBe(`${nodes[0]!.content_fingerprint}#1`); + expect(nodes[1]!.portable_paragraph_ref).toBe(`${nodes[1]!.content_fingerprint}#2`); + }); + }, + ); + + test.openspec('counts are document-wide across paginated windows')( + 'counts are document-wide across paginated windows', + async ({ given, when, then }: AllureBddContext) => { + const dup = 'Repeated boilerplate clause.'; + const mgr = createTestSessionManager(); + const opened = await given('a DOCX with a paragraph repeated three times', () => + openSession([dup, 'Middle one.', dup, 'Middle two.', dup], { mgr }), + ); + + const node = await when( + 'reading only the third duplicate via node_ids', + async () => { + const thirdDupId = opened.paraIds[4]!; + const result = await readFile(mgr, { + file_path: opened.filePath, + format: 'json', + include_fingerprint: true, + include_fingerprint_ordinal: true, + node_ids: [thirdDupId], + }); + assertSuccess(result, 'read'); + const nodes = parseNodes(result.content); + expect(nodes.length).toBe(1); + return nodes[0]!; + }, + ); + + await then( + 'the windowed node reports document-order ordinal 3 and document-wide count 3', + async () => { + expect(node.content_fingerprint_ordinal).toBe(3); + expect(node.content_fingerprint_count_in_document).toBe(3); + }, + ); + }, + ); + + test.openspec('default JSON output omits ordinal fields')( + 'default JSON output omits ordinal fields', + async ({ given, when, then }: AllureBddContext) => { + const mgr = createTestSessionManager(); + const { filePath } = await given('a DOCX session', () => + openSession(['Some text.'], { mgr }), + ); + + const nodes = await when( + 'reading with include_fingerprint but no include_fingerprint_ordinal', + async () => { + const result = await readFile(mgr, { + file_path: filePath, + format: 'json', + include_fingerprint: true, + limit: 100, + }); + assertSuccess(result, 'read'); + return parseNodes(result.content); + }, + ); + + await then('content_fingerprint is present but the ordinal fields are not', async () => { + for (const node of nodes) { + expect(typeof node.content_fingerprint).toBe('string'); + expect(node.content_fingerprint_ordinal).toBeUndefined(); + expect(node.content_fingerprint_count_in_document).toBeUndefined(); + expect(node.portable_paragraph_ref).toBeUndefined(); + } + }); + }, + ); + + test.openspec('TOON format ignores include_fingerprint_ordinal')( + 'TOON format ignores include_fingerprint_ordinal', + async ({ given, when, then }: AllureBddContext) => { + const mgr = createTestSessionManager(); + const { filePath } = await given('a DOCX session', () => + openSession(['Alpha.', 'Alpha.'], { mgr }), + ); + + const baseline = await when('reading TOON with no fingerprint flags', async () => { + const result = await readFile(mgr, { file_path: filePath, format: 'toon', limit: 100 }); + assertSuccess(result, 'read'); + return String(result.content); + }); + + const withFlags = await when( + 'reading TOON with include_fingerprint and include_fingerprint_ordinal', + async () => { + const result = await readFile(mgr, { + file_path: filePath, + format: 'toon', + include_fingerprint: true, + include_fingerprint_ordinal: true, + limit: 100, + }); + assertSuccess(result, 'read'); + return String(result.content); + }, + ); + + await then('TOON output is byte-identical', async () => { + expect(withFlags).toBe(baseline); + }); + }, + ); + + test.openspec('simple format ignores include_fingerprint_ordinal')( + 'simple format ignores include_fingerprint_ordinal', + async ({ given, when, then }: AllureBddContext) => { + const mgr = createTestSessionManager(); + const { filePath } = await given('a DOCX session', () => + openSession(['Alpha.', 'Alpha.'], { mgr }), + ); + + const baseline = await when('reading simple with no fingerprint flags', async () => { + const result = await readFile(mgr, { file_path: filePath, format: 'simple', limit: 100 }); + assertSuccess(result, 'read'); + return String(result.content); + }); + + const withFlags = await when( + 'reading simple with include_fingerprint and include_fingerprint_ordinal', + async () => { + const result = await readFile(mgr, { + file_path: filePath, + format: 'simple', + include_fingerprint: true, + include_fingerprint_ordinal: true, + limit: 100, + }); + assertSuccess(result, 'read'); + return String(result.content); + }, + ); + + await then('simple output is byte-identical', async () => { + expect(withFlags).toBe(baseline); + }); + }, + ); + + test.openspec('Google Docs ignores include_fingerprint_ordinal')( + 'Google Docs ignores include_fingerprint_ordinal', + async ({ given, when, then }: AllureBddContext) => { + // The gdocs read_file handler is a separate code path that builds its own + // nodes; the disambiguation flag must not error and must not add any + // ordinal fields. Mocks mirror the gdocs fingerprint test so this needs no + // real Google Docs credentials. + let manager: SessionManager; + + await given('a mocked Google Docs session', async () => { + vi.doMock('@usejunior/google-docs-core', () => ({ + buildDocumentViewNodes: (paragraphs: any[]) => + paragraphs.map((p: any) => ({ + id: p.anchorId, + list_label: '', + header: '', + style: 'body', + text: p.text, + clean_text: p.text, + tagged_text: p.text, + list_metadata: { + list_level: -1, + label_type: null, + label_string: '', + header_text: null, + header_style: null, + header_formatting: null, + is_auto_numbered: false, + }, + style_fingerprint: { + list_level: -1, + left_indent_pt: 0, + first_line_indent_pt: 0, + style_name: 'body', + alignment: 'LEFT', + }, + paragraph_style_id: null, + paragraph_style_name: 'body', + paragraph_alignment: 'LEFT', + paragraph_indents_pt: { left: 0, first_line: 0 }, + numbering: { num_id: null, ilvl: null, is_auto_numbered: false }, + header_formatting: null, + body_run_formatting: null, + })), + buildParagraphStyleRequest: vi.fn(), + GoogleDocsDocument: { load: vi.fn() }, + isToolSupported: (provider: string, tool: string) => + provider === 'gdocs' && tool === 'read_file', + PROVIDER_CAPABILITIES: { gdocs: new Set(['read_file']) }, + })); + vi.doMock('../gdocs_loader.js', () => ({ + loadGDocsCore: vi.fn(async () => await import('@usejunior/google-docs-core')), + })); + + manager = new SessionManager({ ttlMs: 60_000 }); + const mockParagraphs = [ + { + paragraphId: 'p1', + anchorName: '_bk_0', + anchorId: 't.0:_bk_0', + startIndex: 1, + endIndex: 20, + tabId: 't.0', + text: 'Repeated line', + inTable: false, + }, + { + paragraphId: 'p2', + anchorName: '_bk_1', + anchorId: 't.0:_bk_1', + startIndex: 21, + endIndex: 40, + tabId: 't.0', + text: 'Repeated line', + inTable: false, + }, + ]; + const mockDoc = { + getParagraphs: vi.fn(() => mockParagraphs), + getParagraphTextById: vi.fn(), + getParagraphByAnchorId: vi.fn(), + replaceText: vi.fn(), + insertParagraph: vi.fn(), + executeBatchUpdate: vi.fn(), + exportAsDocx: vi.fn(), + getDocId: vi.fn(() => 'gdocs-test-id'), + getRevisionId: vi.fn(() => 'rev_xyz'), + getEditCount: vi.fn(() => 0), + getEditRevision: vi.fn(() => 0), + getTabs: vi.fn(() => [{ tabId: 't.0', title: 'Tab 1', index: 0 }]), + getDefaultTabId: vi.fn(() => 't.0'), + isRevisionFresh: vi.fn(() => true), + getCache: vi.fn(() => null), + markEdited: vi.fn(), + injectAnchors: vi.fn(), + }; + manager.createGDocsSession('gdocs-test-id', mockDoc as any); + }); + + const result = await when( + 'dispatching read_file with google_doc_id and both fingerprint flags', + async () => { + return dispatchToolCall(manager!, 'read_file', { + google_doc_id: 'gdocs-test-id', + format: 'json', + include_fingerprint: true, + include_fingerprint_ordinal: true, + limit: 100, + }); + }, + ); + + await then('the call succeeds and gdocs nodes carry no ordinal fields', async () => { + expect(result.success, JSON.stringify((result as any).error)).toBe(true); + const nodes = parseNodes((result as any).content); + expect(nodes.length).toBeGreaterThan(0); + for (const node of nodes) { + expect(node.content_fingerprint_ordinal).toBeUndefined(); + expect(node.content_fingerprint_count_in_document).toBeUndefined(); + expect(node.portable_paragraph_ref).toBeUndefined(); + } + }); + + vi.doUnmock('@usejunior/google-docs-core'); + vi.doUnmock('../gdocs_loader.js'); + }, + ); +});