diff --git a/CHANGELOG.md b/CHANGELOG.md index cf00920e..87d5243b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ a promise of ongoing maintenance. ## Unreleased +- Make full Gmail sweeps start from paginated Inbox label-ID enumeration, require every message to + resolve to a conversation, and reject checkpoints that do not read or explicitly carry forward + every resulting thread. - Separate local card dismissal from source cleanup. `tend cli card:dismiss` now moves a reviewable card to `done` with no work item, approval digest, `action:verify`, or connector call, and is reversible with `card:return-to-review`. Explicit source cleanup moves to diff --git a/docs/AGENT_CONTRACT.md b/docs/AGENT_CONTRACT.md index 596015d7..4dc3e84f 100644 --- a/docs/AGENT_CONTRACT.md +++ b/docs/AGENT_CONTRACT.md @@ -65,6 +65,10 @@ For `sweep_rejudge` work: For `recollect_sources` work: - Record source runs with `tend cli source:record-run --work `. +- A full Gmail run must begin with paginated `gmail_search_email_ids(query: "", label_ids: + ["INBOX"])`. Its checkpoint's `inboxEnumeration.messages` must map every authoritative + `messageId` to its direct-read `threadId`; every resulting conversation must appear exactly once + in `readThreadIds` or `carriedForwardThreadIds` before recording the batch. - Record the resulting sweep with `tend cli sweep:record-batch --work `. - Complete the work only after the source run and sweep batch are written back. @@ -89,7 +93,7 @@ Run `tend cli help` for the full command surface. Core feed-runner commands are: | Record heartbeat install | `tend cli feed:heartbeat:installed --feed --automation ` | | Add source | `tend cli source:add --feed --brief ` | | Remove source | `tend cli source:remove --feed --source ` | -| Record source run | `tend cli source:record-run --feed --source --snapshots --judgments --checkpoint [--context-use-file ]` | +| Record source run | `tend cli source:record-run --feed --source (--snapshots \| --snapshots-file ) (--judgments \| --judgments-file ) (--checkpoint \| --checkpoint-file ) [--context-use-file ]` | | Record sweep batch | `tend cli sweep:record-batch --feed --runs [--context ]` | | Record sweep rejudgment | `tend cli sweep:rejudge --feed --feedback --ordered-cards --removed-cards ` | | Upsert card | `tend cli card:upsert --feed --card ` | diff --git a/docs/SKILL.md b/docs/SKILL.md index 809f48ee..8224ffdc 100644 --- a/docs/SKILL.md +++ b/docs/SKILL.md @@ -66,6 +66,10 @@ waking this same thread and saying `go deal with the feed`. 8. For source recollection, record source runs and a sweep batch with the claimed `--work` id before completing the work. If context influenced collection, include a file-backed `contextUse` on the relevant source run and pin the same update id to the sweep batch. + A full Gmail sweep must begin with paginated `gmail_search_email_ids(query: "", label_ids: + ["INBOX"])`. Its checkpoint's `inboxEnumeration.messages` must map every authoritative + `messageId` to its direct-read `threadId`; every resulting conversation must then appear exactly + once in `readThreadIds` or `carriedForwardThreadIds`. Search results alone never define the Inbox. 9. Repeat until `work:claim` returns idle. 10. If a meaningful sweep or refresh happened, ask whether to compound learnings. diff --git a/scripts/smoke-binary.ts b/scripts/smoke-binary.ts index a0bee050..a9215f14 100644 --- a/scripts/smoke-binary.ts +++ b/scripts/smoke-binary.ts @@ -131,7 +131,7 @@ async function validateCliContract(): Promise<{ "feed:drain-agent --feed --agent ", "work:complete --feed --work --token --result ", "card:upsert --feed (--card | --card-file )", - "source:record-run --feed --source --snapshots --judgments --checkpoint [--work ] [--context-use | --context-use-file ]", + "source:record-run --feed --source (--snapshots | --snapshots-file ) (--judgments | --judgments-file ) (--checkpoint | --checkpoint-file ) [--work ] [--context-use | --context-use-file ]", "sweep:record-batch --feed --runs [--work ] [--context ]", "learning:request --feed ", ]; diff --git a/server/cli/contract.ts b/server/cli/contract.ts index 2c9ffe82..a5494fc4 100644 --- a/server/cli/contract.ts +++ b/server/cli/contract.ts @@ -15,7 +15,7 @@ export const CLI_COMMANDS = [ "feed:heartbeat:installed --feed --automation ", "source:add --feed --brief ", "source:remove --feed --source ", - "source:record-run --feed --source --snapshots --judgments --checkpoint [--work ] [--context-use | --context-use-file ]", + "source:record-run --feed --source (--snapshots | --snapshots-file ) (--judgments | --judgments-file ) (--checkpoint | --checkpoint-file ) [--work ] [--context-use | --context-use-file ]", "sweep:record-batch --feed --runs [--work ] [--context ]", "sweep:rejudge --feed --feedback --ordered-cards --removed-cards ", "source:import-json-file --feed --source --path ", diff --git a/server/cli/operator.ts b/server/cli/operator.ts index e489c856..f12f219c 100644 --- a/server/cli/operator.ts +++ b/server/cli/operator.ts @@ -137,9 +137,9 @@ export async function runOperatorCli(rawArgs: string[]): Promise { output = await domain.recordSourceRun( required("feed"), required("source"), - json(required("snapshots")), - json(required("judgments")), - json(required("checkpoint")), + await structured("snapshots"), + await structured("judgments"), + await structured("checkpoint"), value("work"), value("context-use") || value("context-use-file") ? await structured("context-use") diff --git a/server/domain.ts b/server/domain.ts index 14f1cdae..05143a74 100644 --- a/server/domain.ts +++ b/server/domain.ts @@ -176,6 +176,85 @@ function hasText(value: unknown): value is string { return typeof value === "string" && Boolean(value.trim()); } +function sourceRunIdList(value: unknown, label: string): string[] { + if (!Array.isArray(value) || value.some((item) => !hasText(item))) { + throw new Error(`${label} must be an array of non-empty IDs.`); + } + const normalized = value.map((item) => (item as string).trim()); + if (new Set(normalized).size !== normalized.length) throw new Error(`${label} must contain unique IDs.`); + return normalized; +} + +function sourceRunCount(value: unknown, label: string): number { + if (!Number.isInteger(value) || (value as number) < 0) throw new Error(`${label} must be a nonnegative integer.`); + return value as number; +} + +function sourceRunMessageThreads(value: unknown): Array<{ messageId: string; threadId: string }> { + if (!Array.isArray(value)) throw new Error("Full Gmail inboxEnumeration.messages must be an array."); + const messages = value.map((item, index) => { + if (!isRecord(item) || !hasText(item.messageId) || !hasText(item.threadId)) { + throw new Error(`Full Gmail inboxEnumeration.messages item ${index + 1} must contain non-empty messageId and threadId strings.`); + } + return { messageId: item.messageId.trim(), threadId: item.threadId.trim() }; + }); + if (new Set(messages.map((item) => item.messageId)).size !== messages.length) { + throw new Error("Full Gmail inboxEnumeration.messages must contain each messageId exactly once."); + } + return messages; +} + +function assertGmailInboxSweepEnumeration(sourceId: string, checkpoint: unknown): void { + if (sourceId !== "gmail-inbox" || !isRecord(checkpoint)) return; + const fullSweep = checkpoint.fullSweep === true + || (typeof checkpoint.source === "string" && /full[_ -]?inbox[_ -]?sweep/i.test(checkpoint.source)); + if (!fullSweep) return; + + if (!isRecord(checkpoint.inboxEnumeration)) { + throw new Error( + "A full Gmail sweep must begin with an inboxEnumeration manifest from paginated gmail_search_email_ids(query='', label_ids=['INBOX']); search result counts are not authoritative.", + ); + } + const enumeration = checkpoint.inboxEnumeration; + if (enumeration.method !== "gmail_search_email_ids" || enumeration.query !== "") { + throw new Error("Full Gmail inboxEnumeration must use method='gmail_search_email_ids' with an empty query."); + } + const labelIds = sourceRunIdList(enumeration.labelIds, "Full Gmail inboxEnumeration.labelIds"); + if (labelIds.length !== 1 || labelIds[0] !== "INBOX") { + throw new Error("Full Gmail inboxEnumeration.labelIds must be exactly ['INBOX']."); + } + + const labelMessageCount = sourceRunCount(enumeration.labelMessageCount, "Full Gmail inboxEnumeration.labelMessageCount"); + const labelThreadCount = sourceRunCount(enumeration.labelThreadCount, "Full Gmail inboxEnumeration.labelThreadCount"); + const messages = sourceRunMessageThreads(enumeration.messages); + const threadIds = [...new Set(messages.map((item) => item.threadId))]; + const readThreadIds = sourceRunIdList(enumeration.readThreadIds, "Full Gmail inboxEnumeration.readThreadIds"); + const carriedForwardThreadIds = sourceRunIdList(enumeration.carriedForwardThreadIds, "Full Gmail inboxEnumeration.carriedForwardThreadIds"); + + if (messages.length !== labelMessageCount) { + throw new Error(`Full Gmail sweep is incomplete: Inbox reports ${labelMessageCount} messages, but only ${messages.length} authoritative message IDs were resolved to conversations.`); + } + if (threadIds.length !== labelThreadCount) { + throw new Error(`Full Gmail sweep is incomplete: Inbox reports ${labelThreadCount} threads, but direct reads resolved ${threadIds.length}.`); + } + + const enumeratedThreads = new Set(threadIds); + const readThreads = new Set(readThreadIds); + const carriedThreads = new Set(carriedForwardThreadIds); + const unknownThreads = [...readThreads, ...carriedThreads].filter((threadId) => !enumeratedThreads.has(threadId)); + if (unknownThreads.length) { + throw new Error(`Full Gmail inboxEnumeration dispositions include threads outside the authoritative manifest: ${unknownThreads.join(", ")}.`); + } + const overlaps = readThreadIds.filter((threadId) => carriedThreads.has(threadId)); + if (overlaps.length) { + throw new Error(`Full Gmail inboxEnumeration must classify each thread once; these are both read and carried forward: ${overlaps.join(", ")}.`); + } + const unresolvedThreads = threadIds.filter((threadId) => !readThreads.has(threadId) && !carriedThreads.has(threadId)); + if (unresolvedThreads.length) { + throw new Error(`Full Gmail sweep is incomplete: ${unresolvedThreads.length} authoritative Inbox thread(s) were neither read nor explicitly carried forward: ${unresolvedThreads.join(", ")}.`); + } +} + function isSafeCardHref(value: string): boolean { if (value.startsWith("/api/artifacts/")) return true; try { @@ -2686,6 +2765,7 @@ export class AttentionDomain { const feed = await this.store.readFeed(feedId); if (!feed.sources.some((source) => source.id === sourceId)) throw new Error(`Source recipe not found: ${sourceId}`); if (triggerWorkId) await this.assertClaimedRecollectionWork(feedId, triggerWorkId); + assertGmailInboxSweepEnumeration(sourceId, checkpoint); const normalizedContextUse = contextUse ? normalizeContextUse(contextUse, await this.requireCurrentMindContext(contextUse.updateId), snapshots) : undefined; diff --git a/server/operator.ts b/server/operator.ts index 26c0cea5..2538ff59 100644 --- a/server/operator.ts +++ b/server/operator.ts @@ -240,7 +240,9 @@ export function formatWorkClaimOutput(feedId: string, work: WorkClaimResult, con if (work.intent === "recollect_sources") { operatorGuidance.requiredWriteBack = "Record one or more source runs with `source:record-run --work `, then create a sweep batch with `sweep:record-batch --work ` before `work:complete`."; - operatorGuidance.sourceRunRule = "Source recollection work must complete with a new sweep batch recorded for this exact work item."; + operatorGuidance.sourceRunRule = feedId === "inbox" + ? "For a full Gmail sweep, first paginate gmail_search_email_ids(query='', label_ids=['INBOX']). Treat that message-ID manifest as authoritative, direct-read every ID, and record an inboxEnumeration.messages entry mapping each messageId to its threadId. Its readThreadIds and carriedForwardThreadIds must then classify every resulting thread exactly once. gmail_search_emails results may enrich the run but cannot define the Inbox universe. Source recollection must complete with a new sweep batch recorded for this exact work item." + : "Source recollection work must complete with a new sweep batch recorded for this exact work item."; } return Object.keys(operatorGuidance).length ? { ...work, operatorGuidance } : work; diff --git a/server/templates.ts b/server/templates.ts index e62be510..d42d5d81 100644 --- a/server/templates.ts +++ b/server/templates.ts @@ -130,6 +130,17 @@ thread before composing a card. Treat Gmail's subject and latest snippet as evid concrete event, decision, or request, and summarize that plainly instead of pasting reply-chain fragments. +For a full sweep, begin by paginating \`gmail_search_email_ids(query: "", label_ids: ["INBOX"])\` +until it returns no next page. That complete message-ID list is the authoritative Inbox manifest; +\`gmail_search_emails(query: "in:inbox")\` may enrich the run, but its result set must never define +the sweep universe. Direct-read every enumerated message and map it to its conversation. Record +\`fullSweep: true\` and an \`inboxEnumeration\` containing \`method\`, empty \`query\`, \`labelIds\`, +\`labelMessageCount\`, \`labelThreadCount\`, a \`messages\` array mapping every \`messageId\` to its +\`threadId\`, \`readThreadIds\`, and \`carriedForwardThreadIds\`. Tend derives the conversation +universe from those mappings and accepts the checkpoint only when manifest counts match and every +conversation has exactly one disposition. Prefer \`source:record-run --snapshots-file ... +--judgments-file ... --checkpoint-file ...\` for these larger structured payloads. + Separate conservative low-attention cleanup into a proposed \`routine_action\` group such as \`Likely archive\`. Keep requests, ambiguous threads, and anything with a meaningful next move as full review cards. The group is an approval surface, not permission to archive automatically. diff --git a/test/cli-contract.test.ts b/test/cli-contract.test.ts index aead5184..2bd545ae 100644 --- a/test/cli-contract.test.ts +++ b/test/cli-contract.test.ts @@ -48,6 +48,18 @@ describe("CLI contract", () => { for (const command of documented) expect(commandNames).toContain(command); }); + test("source:record-run supports file-backed full-sweep manifests", async () => { + const command = CLI_COMMANDS.find((candidate) => cliCommandName(candidate) === "source:record-run"); + expect(command).toContain("--snapshots-file "); + expect(command).toContain("--judgments-file "); + expect(command).toContain("--checkpoint-file "); + + const operator = await readFile("server/cli/operator.ts", "utf8"); + expect(operator).toContain('await structured("snapshots")'); + expect(operator).toContain('await structured("judgments")'); + expect(operator).toContain('await structured("checkpoint")'); + }); + test("formats command-owned usage hints for missing flags", () => { const error = formatCliError(new MissingFlagError("work:claim", "thread")); diff --git a/test/domain.test.ts b/test/domain.test.ts index 1e041220..b7aac8fa 100644 --- a/test/domain.test.ts +++ b/test/domain.test.ts @@ -133,6 +133,17 @@ describe("feed thread operator handshake", () => { }); }); + test("gives Inbox recollection claims the authoritative label-ID collection order", () => { + const work = { id: "work-1", intent: "recollect_sources" } as WorkItem; + const output = formatWorkClaimOutput("inbox", work); + + expect(output).toMatchObject({ + operatorGuidance: { + sourceRunRule: expect.stringMatching(/first paginate gmail_search_email_ids.*cannot define the Inbox universe/), + }, + }); + }); + test("includes a click authorization receipt on claimed approved action work", async () => { const { store, domain } = await setup(); await domain.bindFeed("inbox", "thread-inbox"); @@ -361,6 +372,71 @@ describe("filesystem workspace", () => { expect((await store.readSweepState("inbox")).currentBatchId).toBe(batchId); }); + test("requires full Gmail sweeps to start from an authoritative Inbox ID manifest", async () => { + const { domain, store } = await setup(); + const checkpointBefore = await store.readSourceCheckpoint("inbox", "gmail-inbox"); + await expect(domain.recordSourceRun("inbox", "gmail-inbox", [{ threads: [] }], [], { + source: "gmail_connector", + fullSweep: true, + labelThreadCount: 45, + enumeratedThreadCount: 40, + carriedForwardThreadIds: ["one", "two", "three", "four", "five"], + })).rejects.toThrow("must begin with an inboxEnumeration manifest"); + expect(await store.readSourceCheckpoint("inbox", "gmail-inbox")).toEqual(checkpointBefore); + expect((await store.readEvents("inbox")).filter((event) => event.type === "source.run_completed")).toHaveLength(0); + }); + + test("rejects incomplete full Gmail ID manifests and accepts complete per-thread dispositions", async () => { + const { domain } = await setup(); + const inboxEnumeration = { + method: "gmail_search_email_ids", + query: "", + labelIds: ["INBOX"], + labelMessageCount: 4, + labelThreadCount: 3, + messages: [ + { messageId: "message-1", threadId: "thread-1" }, + { messageId: "message-2", threadId: "thread-1" }, + { messageId: "message-3", threadId: "thread-2" }, + { messageId: "message-4", threadId: "thread-3" }, + ], + readThreadIds: ["thread-1"], + carriedForwardThreadIds: ["thread-2"], + }; + await expect(domain.recordSourceRun("inbox", "gmail-inbox", [{ threads: [] }], [], { + source: "gmail_connector_full_inbox_sweep", + fullSweep: true, + inboxEnumeration: { + ...inboxEnumeration, + messages: inboxEnumeration.messages.slice(0, 3), + }, + })).rejects.toThrow("Inbox reports 4 messages, but only 3 authoritative message IDs were resolved to conversations"); + + await expect(domain.recordSourceRun("inbox", "gmail-inbox", [{ threads: [] }], [], { + source: "gmail_connector_full_inbox_sweep", + fullSweep: true, + inboxEnumeration: { + ...inboxEnumeration, + messages: [inboxEnumeration.messages[0], inboxEnumeration.messages[0], ...inboxEnumeration.messages.slice(2)], + }, + })).rejects.toThrow("must contain each messageId exactly once"); + + await expect(domain.recordSourceRun("inbox", "gmail-inbox", [{ threads: [] }], [], { + source: "gmail_connector_full_inbox_sweep", + fullSweep: true, + inboxEnumeration, + })).rejects.toThrow("1 authoritative Inbox thread(s) were neither read nor explicitly carried forward"); + + await expect(domain.recordSourceRun("inbox", "gmail-inbox", [{ threads: [] }], [], { + source: "gmail_connector_full_inbox_sweep", + fullSweep: true, + inboxEnumeration: { + ...inboxEnumeration, + carriedForwardThreadIds: ["thread-2", "thread-3"], + }, + })).resolves.toMatch(/^run_/); + }); + test("rejects source-backed card writes and actions from stale sweep runs", async () => { const { domain, store } = await setup(); const oldRun = await domain.recordSourceRun("inbox", "gmail-inbox", [{ threadId: "gmail-old", subject: "Sign this" }], [{ decision: "keep" }], { cursor: "gmail-old" });