From d5dde3eb1c1063c09f45302436eeb81b261a51a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 22 Aug 2026 09:56:10 +0200 Subject: [PATCH 1/2] Carry the vault's filing and its staged facts through backup and restore `DocumentConditionLink` and `ExtractedFact` were classified as backed up and came back from no restore. The register said a restored vault is unsorted and that the facts are re-derivable by re-running the extraction. The first is right; the second understates it. What a re-run cannot produce is the REVIEW DECISION on each fact, and every column holding one defaults to "nobody has looked at this yet". `POST /api/documents/inbound/[id]/confirm` acts only on a PENDING fact and commits it into the structured store, so a restore that let those columns default would offer an already-committed reading for approval a second time and write a second lab result behind it. Both ends live in `src/lib/export/document-filing-backup.ts`, beside each other like the Coach, reminder and vaccination sections. `status`, `needsReview`, `confidence` and the commitment pair are carried and restored verbatim, and the wire schema requires the first three rather than defaulting them. Ordering: the section runs after the documents, the condition episodes, the lab results and the medications. The first two are real foreign keys, unlike the Coach's bare id columns, so an unresolvable value there does not cost one edge, it aborts the transaction and the operator gets none of the account back. The last two are what `committedRecordId` resolves against. The builder therefore carries a filing or a fact only when both of its ends are carried, and the restore still checks and drops what a hand-edited file leaves unresolvable. `committedRecordId` is nulled together with its type, because a type with no id claims a commitment the row cannot name. The round trip seeds a second condition and files the page under it, so a restore that kept the count and lost the pairing fails rather than handing back a lab report filed under a head cold. A third case breaks both references in a saved file to prove the drop path runs and still answers 200. Three new skip catalogues arrive with it, and with them labels for the three that had none: a dropped Coach attachment had been reaching the operator's screen labelled "mood factor" since it was added. The label chain is exhaustive now, so the next catalogue without one stops the build. --- messages/de.json | 6 + messages/en.json | 6 + messages/es.json | 6 + messages/fr.json | 6 + messages/it.json | 6 + messages/pl.json | 6 + .../[id]/restore/__tests__/round-trip.test.ts | 4 + .../api/admin/backups/[id]/restore/route.ts | 41 +- .../export/__tests__/per-type-routes.test.ts | 8 + .../__tests__/soft-delete-filter.test.ts | 7 + .../export/encrypted/__tests__/route.test.ts | 4 + src/components/admin/backups-section.tsx | 29 +- .../__tests__/full-backup-payload.test.ts | 4 + src/lib/export/backup-plan.ts | 40 +- src/lib/export/document-filing-backup.ts | 444 ++++++++++++++++++ src/lib/export/full-backup-payload.ts | 22 +- src/lib/export/restore-skips.ts | 23 +- .../__tests__/link-surface-guard.test.ts | 20 +- src/lib/validations/backup.ts | 61 +++ tests/integration/backup-round-trip.test.ts | 373 ++++++++++++++- 20 files changed, 1084 insertions(+), 32 deletions(-) create mode 100644 src/lib/export/document-filing-backup.ts diff --git a/messages/de.json b/messages/de.json index d835c5c7f..93a4690e0 100644 --- a/messages/de.json +++ b/messages/de.json @@ -6912,6 +6912,12 @@ "restoreSkippedVisitReference": "Besuchsverknüpfung", "restoreSkippedVaccinationReference": "Impfverknüpfung", "restoreSkippedCheckupClosure": "Vorsorge-Abschluss", + "restoreSkippedReminderReference": "Erinnerungsverknüpfung", + "restoreSkippedCoachAttachment": "Coach-Dokumentverknüpfung", + "restoreSkippedCoachReference": "Coach-Verweis", + "restoreSkippedDocumentConditionLink": "Dokumentzuordnung", + "restoreSkippedExtractedFact": "Dokumentbefund", + "restoreSkippedFactCommitment": "Befundübernahme", "docsLink": "Dokumentation" }, "danger-zone": { diff --git a/messages/en.json b/messages/en.json index 10d6dd7a1..3ff9c8057 100644 --- a/messages/en.json +++ b/messages/en.json @@ -6912,6 +6912,12 @@ "restoreSkippedVisitReference": "Visit reference", "restoreSkippedVaccinationReference": "Vaccination reference", "restoreSkippedCheckupClosure": "Checkup closure", + "restoreSkippedReminderReference": "Reminder reference", + "restoreSkippedCoachAttachment": "Coach document link", + "restoreSkippedCoachReference": "Coach reference", + "restoreSkippedDocumentConditionLink": "Document filing", + "restoreSkippedExtractedFact": "Document fact", + "restoreSkippedFactCommitment": "Fact commitment", "docsLink": "Documentation" }, "danger-zone": { diff --git a/messages/es.json b/messages/es.json index a42325704..b0ef97c44 100644 --- a/messages/es.json +++ b/messages/es.json @@ -6912,6 +6912,12 @@ "restoreSkippedVisitReference": "Referencia de visita", "restoreSkippedVaccinationReference": "Referencia de vacunación", "restoreSkippedCheckupClosure": "Cierre de revisión", + "restoreSkippedReminderReference": "Referencia de recordatorio", + "restoreSkippedCoachAttachment": "Enlace de documento del Coach", + "restoreSkippedCoachReference": "Referencia del Coach", + "restoreSkippedDocumentConditionLink": "Clasificación de documento", + "restoreSkippedExtractedFact": "Dato del documento", + "restoreSkippedFactCommitment": "Registro del dato", "docsLink": "Documentación" }, "danger-zone": { diff --git a/messages/fr.json b/messages/fr.json index 4374d6dd9..b0477629c 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -6912,6 +6912,12 @@ "restoreSkippedVisitReference": "Référence de visite", "restoreSkippedVaccinationReference": "Référence de vaccination", "restoreSkippedCheckupClosure": "Clôture d'un examen", + "restoreSkippedReminderReference": "Référence de rappel", + "restoreSkippedCoachAttachment": "Lien de document du Coach", + "restoreSkippedCoachReference": "Référence du Coach", + "restoreSkippedDocumentConditionLink": "Classement du document", + "restoreSkippedExtractedFact": "Donnée du document", + "restoreSkippedFactCommitment": "Enregistrement de la donnée", "docsLink": "Documentation" }, "danger-zone": { diff --git a/messages/it.json b/messages/it.json index aeabdc9ab..85815b598 100644 --- a/messages/it.json +++ b/messages/it.json @@ -6912,6 +6912,12 @@ "restoreSkippedVisitReference": "Riferimento alla visita", "restoreSkippedVaccinationReference": "Riferimento alla vaccinazione", "restoreSkippedCheckupClosure": "Chiusura del controllo", + "restoreSkippedReminderReference": "Riferimento al promemoria", + "restoreSkippedCoachAttachment": "Collegamento documento del Coach", + "restoreSkippedCoachReference": "Riferimento del Coach", + "restoreSkippedDocumentConditionLink": "Classificazione del documento", + "restoreSkippedExtractedFact": "Dato del documento", + "restoreSkippedFactCommitment": "Registrazione del dato", "docsLink": "Documentazione" }, "danger-zone": { diff --git a/messages/pl.json b/messages/pl.json index 3b5eff3c7..a712d08d7 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -6912,6 +6912,12 @@ "restoreSkippedVisitReference": "Odwołanie wizyty", "restoreSkippedVaccinationReference": "Odwołanie szczepienia", "restoreSkippedCheckupClosure": "Zamknięcie badania", + "restoreSkippedReminderReference": "Odwołanie do przypomnienia", + "restoreSkippedCoachAttachment": "Powiązanie dokumentu z Coachem", + "restoreSkippedCoachReference": "Odwołanie Coacha", + "restoreSkippedDocumentConditionLink": "Przypisanie dokumentu", + "restoreSkippedExtractedFact": "Fakt z dokumentu", + "restoreSkippedFactCommitment": "Zapis faktu", "docsLink": "Dokumentacja" }, "danger-zone": { diff --git a/src/app/api/admin/backups/[id]/restore/__tests__/round-trip.test.ts b/src/app/api/admin/backups/[id]/restore/__tests__/round-trip.test.ts index 1b26eb351..89bd12f65 100644 --- a/src/app/api/admin/backups/[id]/restore/__tests__/round-trip.test.ts +++ b/src/app/api/admin/backups/[id]/restore/__tests__/round-trip.test.ts @@ -304,6 +304,10 @@ function sourceClient() { coachFact: { findMany: vi.fn().mockResolvedValue([]) }, coachPlan: { findMany: vi.fn().mockResolvedValue([]) }, coachReminder: { findMany: vi.fn().mockResolvedValue([]) }, + // What a document was filed against, and what was read out of it. + // Empty for the same reason as the sections above. + documentConditionLink: { findMany: vi.fn().mockResolvedValue([]) }, + extractedFact: { findMany: vi.fn().mockResolvedValue([]) }, customMetric: { findMany: vi.fn().mockResolvedValue([ { diff --git a/src/app/api/admin/backups/[id]/restore/route.ts b/src/app/api/admin/backups/[id]/restore/route.ts index 2449a082c..a6d1f28d8 100644 --- a/src/app/api/admin/backups/[id]/restore/route.ts +++ b/src/app/api/admin/backups/[id]/restore/route.ts @@ -56,6 +56,7 @@ import { restoreCoachMemoryData, } from "@/lib/export/coach-backup"; import { restoreRemindersData } from "@/lib/export/reminders-backup"; +import { restoreDocumentFilingData } from "@/lib/export/document-filing-backup"; import { invalidateUserData } from "@/lib/cache/invalidate"; export const dynamic = "force-dynamic"; @@ -90,6 +91,8 @@ interface RestoreResponse { familyHistory: number; workouts: number; documents: number; + documentConditionLinks: number; + extractedFacts: number; healthProfile: number; healthProfileFactRevisions: number; customMetrics: number; @@ -1215,6 +1218,10 @@ const handler = apiHandler( restoredBiomarkerIds.add(created.id); } + // Collected for the staged facts further down: an approved fact's + // `committedRecordId` names the lab result it was committed to, and + // that reference is resolved against the rows this loop writes. + const restoredLabResultIds = new Set(); for (const lab of payload.labResults) { const biomarkerId = lab.biomarkerId !== undefined @@ -1231,7 +1238,7 @@ const handler = apiHandler( `Unknown biomarker reference: ${lab.biomarkerId ?? lab.biomarkerName}`, ); } - await tx.labResult.create({ + const createdLab = await tx.labResult.create({ data: { ...(lab.id ? { id: lab.id } : {}), userId: ownerId, @@ -1265,6 +1272,7 @@ const handler = apiHandler( : {}), }, }); + restoredLabResultIds.add(createdLab.id); } const episodeIds = new Set( @@ -1538,6 +1546,34 @@ const handler = apiHandler( }); } + // What the vault was filed against, and what was read out of it. + // AFTER the documents and the condition episodes because both are + // foreign keys here — a filing written before either exists does not + // drop quietly, it violates a constraint and costs the operator the + // whole restore. AFTER the lab results and the medications for a + // second reason: an approved fact's `committedRecordId` is resolved + // against the rows those branches wrote, and resolving before they + // exist would null every commitment and still report success. Both + // ends of this section live in + // `src/lib/export/document-filing-backup.ts`. + const documentFilingCleared = await restoreDocumentFilingData( + tx, + ownerId, + payload, + { + documentIds: new Set( + payload.documents.map((document) => document.id), + ), + episodeIds, + committedRecordIds: new Set([ + ...restoredLabResultIds, + ...episodeIds, + ...restoredMedicationIds, + ]), + }, + skips, + ); + // The Vorsorge reminders and their completion ledger (v1.37.20, // #223 / iOS #68), BEFORE the visits and the vaccinations: both of // those remap a `reminderId` against the reminders in the database, @@ -1637,6 +1673,9 @@ const handler = apiHandler( familyHistory: familyHistory.count, workouts: workouts.count, documents: documents.count, + documentConditionLinks: + documentFilingCleared.documentConditionLinks, + extractedFacts: documentFilingCleared.extractedFacts, healthProfile: profileCleared.healthProfile, healthProfileFactRevisions: profileCleared.healthProfileFactRevisions, diff --git a/src/app/api/export/__tests__/per-type-routes.test.ts b/src/app/api/export/__tests__/per-type-routes.test.ts index 78be29c14..34dc352e9 100644 --- a/src/app/api/export/__tests__/per-type-routes.test.ts +++ b/src/app/api/export/__tests__/per-type-routes.test.ts @@ -50,6 +50,10 @@ vi.mock("@/lib/db", () => ({ coachFact: { findMany: vi.fn().mockResolvedValue([]) }, coachPlan: { findMany: vi.fn().mockResolvedValue([]) }, coachReminder: { findMany: vi.fn().mockResolvedValue([]) }, + // What a document was filed against, and what was read out of it. + // Empty for the same reason as the sections above. + documentConditionLink: { findMany: vi.fn().mockResolvedValue([]) }, + extractedFact: { findMany: vi.fn().mockResolvedValue([]) }, // v1.15.0 — cycle tables read by the full-backup helper. cycleProfile: { findUnique: vi.fn() }, menstrualCycle: { findMany: vi.fn() }, @@ -147,6 +151,10 @@ beforeEach(() => { [] as never, ); vi.mocked(prisma.consentReceipt.findMany).mockResolvedValue([] as never); + vi.mocked(prisma.documentConditionLink.findMany).mockResolvedValue( + [] as never, + ); + vi.mocked(prisma.extractedFact.findMany).mockResolvedValue([] as never); }); afterEach(() => { diff --git a/src/app/api/export/__tests__/soft-delete-filter.test.ts b/src/app/api/export/__tests__/soft-delete-filter.test.ts index da1e8c227..40169cfe2 100644 --- a/src/app/api/export/__tests__/soft-delete-filter.test.ts +++ b/src/app/api/export/__tests__/soft-delete-filter.test.ts @@ -60,6 +60,9 @@ vi.mock("@/lib/db", () => ({ familyHistoryEntry: { findMany: vi.fn() }, workout: { findMany: vi.fn() }, inboundDocument: { findMany: vi.fn() }, + // What a document was filed against, and what was read out of it. + documentConditionLink: { findMany: vi.fn() }, + extractedFact: { findMany: vi.fn() }, }, })); @@ -167,6 +170,10 @@ beforeEach(() => { vi.mocked(prisma.familyHistoryEntry.findMany).mockResolvedValue([] as never); vi.mocked(prisma.workout.findMany).mockResolvedValue([] as never); vi.mocked(prisma.inboundDocument.findMany).mockResolvedValue([] as never); + vi.mocked(prisma.documentConditionLink.findMany).mockResolvedValue( + [] as never, + ); + vi.mocked(prisma.extractedFact.findMany).mockResolvedValue([] as never); }); describe("v1.4.41 W-DELETED-2 — soft-delete invisibility", () => { diff --git a/src/app/api/export/encrypted/__tests__/route.test.ts b/src/app/api/export/encrypted/__tests__/route.test.ts index dc1007b6c..7e86387a3 100644 --- a/src/app/api/export/encrypted/__tests__/route.test.ts +++ b/src/app/api/export/encrypted/__tests__/route.test.ts @@ -48,6 +48,10 @@ vi.mock("@/lib/db", () => ({ coachFact: { findMany: vi.fn().mockResolvedValue([]) }, coachPlan: { findMany: vi.fn().mockResolvedValue([]) }, coachReminder: { findMany: vi.fn().mockResolvedValue([]) }, + // What a document was filed against, and what was read out of it. + // Empty for the same reason as the sections above. + documentConditionLink: { findMany: vi.fn().mockResolvedValue([]) }, + extractedFact: { findMany: vi.fn().mockResolvedValue([]) }, cycleProfile: { findUnique: vi.fn().mockResolvedValue(null) }, menstrualCycle: { findMany: vi.fn().mockResolvedValue([]) }, cycleDayLog: { findMany: vi.fn().mockResolvedValue([]) }, diff --git a/src/components/admin/backups-section.tsx b/src/components/admin/backups-section.tsx index abcf78d0e..e63ffde07 100644 --- a/src/components/admin/backups-section.tsx +++ b/src/components/admin/backups-section.tsx @@ -332,7 +332,34 @@ function catalogueLabel( if (catalogue === "checkupClosure") { return t("admin.section.backups.restoreSkippedCheckupClosure"); } - return t("admin.section.backups.restoreSkippedMoodFactor"); + if (catalogue === "reminderReference") { + return t("admin.section.backups.restoreSkippedReminderReference"); + } + if (catalogue === "coachAttachment") { + return t("admin.section.backups.restoreSkippedCoachAttachment"); + } + if (catalogue === "coachReference") { + return t("admin.section.backups.restoreSkippedCoachReference"); + } + if (catalogue === "documentConditionLink") { + return t("admin.section.backups.restoreSkippedDocumentConditionLink"); + } + if (catalogue === "extractedFact") { + return t("admin.section.backups.restoreSkippedExtractedFact"); + } + if (catalogue === "factCommitment") { + return t("admin.section.backups.restoreSkippedFactCommitment"); + } + if (catalogue === "moodFactor") { + return t("admin.section.backups.restoreSkippedMoodFactor"); + } + // Not a fallback: the chain above is exhaustive and this line is what makes + // the compiler say so. A catalogue added without a label here now stops the + // build instead of shipping under a label that belongs to something else, + // which is what three of them were already doing — a dropped Coach + // attachment reached the operator's screen labelled "mood factor". + const unlabelled: never = catalogue; + return unlabelled; } /** diff --git a/src/lib/export/__tests__/full-backup-payload.test.ts b/src/lib/export/__tests__/full-backup-payload.test.ts index 8dbd3c684..7537e3a92 100644 --- a/src/lib/export/__tests__/full-backup-payload.test.ts +++ b/src/lib/export/__tests__/full-backup-payload.test.ts @@ -363,6 +363,10 @@ function makePrisma() { coachFact: { findMany: vi.fn().mockResolvedValue([]) }, coachPlan: { findMany: vi.fn().mockResolvedValue([]) }, coachReminder: { findMany: vi.fn().mockResolvedValue([]) }, + // What a document was filed against, and what was read out of it. + // Empty for the same reason as the sections above. + documentConditionLink: { findMany: vi.fn().mockResolvedValue([]) }, + extractedFact: { findMany: vi.fn().mockResolvedValue([]) }, // Left unmocked on purpose: `buildProfileBackupSection` runs for real // against these, so the assertions below exercise the builder rather than // a stand-in that would agree with whatever the payload happened to do. diff --git a/src/lib/export/backup-plan.ts b/src/lib/export/backup-plan.ts index 05cee6527..797c3a630 100644 --- a/src/lib/export/backup-plan.ts +++ b/src/lib/export/backup-plan.ts @@ -226,6 +226,10 @@ export const BACKUP_WRITER_FILES: readonly string[] = [ // The screener history and the consent record, disaster-recovery only. The // reasons for that live in the module itself. "src/lib/export/sensitive-backup.ts", + // What a document was filed against and what was read out of it. The + // documents themselves are read by `records-backup.ts`; these two read + // through their OWN delegates here, for the reason the visits comment gives. + "src/lib/export/document-filing-backup.ts", "src/lib/cycle/backup.ts", ]; @@ -239,6 +243,7 @@ export const BACKUP_RESTORE_FILES: readonly string[] = [ "src/lib/export/reminders-backup.ts", "src/lib/export/coach-backup.ts", "src/lib/export/sensitive-backup.ts", + "src/lib/export/document-filing-backup.ts", "src/lib/cycle/backup.ts", ]; @@ -304,12 +309,13 @@ export const TWO_ENDED_MODELS = [ "EncounterLabLink", "EncounterConditionLink", // Doses travel both ways from the release that introduces them, and the - // link with them. `DocumentConditionLink` one list down says what the - // alternative costs — documents and conditions both restore, the filing - // between them does not. Repeating that here would return a restored - // Impfpass scan and a restored dose with nothing between them, which is the - // same regret against a record a person cannot reconstruct from anywhere - // else. + // link with them. `DocumentConditionLink` was the register entry that said + // what the alternative costs — documents and conditions both restore, the + // filing between them does not — and it has since landed carried itself, at + // the bottom of this list. Repeating that debt here would have returned a + // restored Impfpass scan and a restored dose with nothing between them, + // which is the same regret against a record a person cannot reconstruct + // from anywhere else. "VaccinationRecord", "VaccinationDocumentLink", // The Vorsorge reminders, off the debt register at last (v1.37.20, #223 / @@ -388,6 +394,24 @@ export const TWO_ENDED_MODELS = [ // neither. "MentalHealthAssessment", "ConsentReceipt", + // What the vault was sorted into, and what was read out of it. The register + // called the first "a restored vault is unsorted" and the second + // "re-derivable only by re-running the extraction against a provider, at the + // operator's cost". The second understated it: what a re-run cannot produce + // is the REVIEW DECISION on each fact, and every column holding one defaults + // to "nobody has looked at this yet". The confirm endpoint acts on a PENDING + // fact by committing it into the structured store, so a restore that + // defaulted them would offer an already-committed reading for approval a + // second time and write a second lab result behind it. + // + // Both references are real foreign keys, unlike the Coach's, so the failure + // they guard against is not a dead pointer but an aborted transaction: the + // builder carries a filing or a fact only when both of its ends are carried, + // and the restore drops and names what a hand-edited file still cannot + // place. `committedRecordId` is the one bare id column here, and it is + // nulled together with its type when it resolves to nothing. + "DocumentConditionLink", + "ExtractedFact", ] as const; /** One model claimed to travel both ways. */ @@ -442,10 +466,6 @@ export const COVERAGE_PENDING: Readonly> = { "Per-day environmental readings joined to the record. Re-fetchable for recent days only; older history is gone once the provider window closes.", EnvironmentTravelLocation: "Where the person was on a given day, which is what makes the environmental readings mean anything. Never re-derivable.", - DocumentConditionLink: - "Which documents were filed against which condition. Documents and conditions both restore; the filing between them does not, so a restored vault is unsorted.", - ExtractedFact: - "Facts read out of a document by the AI pass, with their provenance back to the page. Re-derivable only by re-running the extraction against a provider, at the operator's cost.", }; /** diff --git a/src/lib/export/document-filing-backup.ts b/src/lib/export/document-filing-backup.ts new file mode 100644 index 000000000..b310fe874 --- /dev/null +++ b/src/lib/export/document-filing-backup.ts @@ -0,0 +1,444 @@ +/** + * What a stored document was filed against, and what was read out of it, with + * both backup ends in one file. + * + * Same arrangement as `coach-backup.ts`, `reminders-backup.ts` and + * `vaccinations-backup.ts`, for the same reason: a reader asking "is this + * carried at both ends?" answers it here, and a reader who greps only the + * restore ROUTE gets a false negative because the route delegates. + * + * Two models ride, and they are the two halves of the same sentence about a + * document. `DocumentConditionLink` says which conditions a page belongs to. + * `ExtractedFact` says what the extraction pass transcribed out of it, with the + * span it came from. Both sat on the coverage-pending register, where the + * entries read "a restored vault is unsorted" and "re-derivable only by + * re-running the extraction against a provider, at the operator's cost". + * + * ## The state on a fact is worth more than the fact + * + * `status`, `needsReview` and the pair `committedRecordId` / + * `committedRecordType` are the record of a DECISION a person already made: + * this fact was reviewed, approved, and committed to that lab result. Every one + * of them has a schema default that reads as "nobody has looked at this yet" + * (`PENDING`, `needsReview: true`, both commitment columns NULL), so a restore + * that let them default hands back the same rows with their history erased. + * + * That is not only lost history. `POST /api/documents/inbound/[id]/confirm` + * accepts a decision only on a `PENDING` fact and commits it through the normal + * field-by-field create — so a fact that comes back PENDING is offered for + * review a second time, and approving it writes a SECOND lab result, condition + * or medication for a reading the account already has. The count of facts would + * be right, the vault would look right, and the structured store would start + * doubling. So the four columns are carried and restored verbatim, and asserted + * by name in the round trip. + * + * ## Two references that behave very differently + * + * `documentId` is a real foreign key on both models, and `episodeId` is one on + * the link. A value that cannot resolve does not quietly stop meaning + * something: it aborts the transaction and costs the operator the WHOLE + * restore. So the builder carries a row only when both of its ends are + * carried — a document tombstoned out of the file takes its filings and its + * facts with it — and the restore still checks against what it actually wrote + * and drops what it cannot place, because a hand-edited or truncated file is + * the case the write-time filter cannot cover. + * + * `committedRecordId` is the opposite: a bare id column with no relation + * declared, pointing into one of three different tables depending on + * `committedRecordType`. Nothing in the database refuses a value that points at + * nothing, which makes it more dangerous rather than less. It is resolved + * against the lab results, condition episodes and medications the restore has + * just written, and a value that resolves to nothing is nulled TOGETHER with + * its type and reported — a fact that still claimed to have been committed + * somewhere, with no somewhere, would send a reader looking for a row that is + * not there. + * + * ## Ciphertext follows the contract every other encrypted column here uses + * + * A disaster-recovery payload carries the stored bytes verbatim as base64, + * because the same instance's key reads them back unchanged. A portable export + * carries the decrypted JSON, because a portable export exists to be readable + * by the person who owns it, exactly as it already decrypts medication notes, + * mood notes and document summaries. + * + * The portable arm of the RESTORE is there for the reader's file, not for a + * path this release can reach: a fact cannot exist without a document, and the + * route refuses a portable file that carries any document ahead of the wipe + * because document ciphertext is not in it. It is written the same way as the + * coach turns rather than as a special case, so the next person reading either + * file finds one contract instead of two. + */ +import type { Prisma, PrismaClient } from "@/generated/prisma/client"; + +import { decryptFromBytes, encryptToBytes } from "@/lib/ai/coach/bytes-codec"; +import { + recordUnknownKeys, + type RestoreSkipLog, +} from "@/lib/export/restore-skips"; + +export interface DocumentFilingBackupOptions { + purpose?: "portable-export" | "disaster-recovery"; +} + +/** One document filed against one condition episode. */ +export interface DocumentConditionLinkBackupEntry { + documentId: string; + episodeId: string; + createdAt: string; +} + +/** One fact the extraction pass staged against a document. */ +export interface ExtractedFactBackupEntry { + id: string; + documentId: string; + factType: string; + /** + * The review state and the commitment it produced. Carried verbatim; see the + * file header for why letting these default re-opens a decided fact and lets + * a second approval double the structured row behind it. + */ + status: string; + confidence: number; + needsReview: boolean; + committedRecordId: string | null; + committedRecordType: string | null; + /** Ciphertext as base64. Present on a disaster-recovery payload only. */ + dataEncrypted?: string; + provenanceEncrypted?: string; + /** Decrypted JSON. Present on a portable payload only. */ + data?: unknown; + provenance?: unknown; + createdAt: string; + updatedAt: string; +} + +export interface DocumentFilingBackupSection { + documentConditionLinks: DocumentConditionLinkBackupEntry[]; + extractedFacts: ExtractedFactBackupEntry[]; +} + +export interface DocumentFilingBackupCounts { + documentConditionLinks: number; + extractedFacts: number; +} + +/** + * Every restorable column of both models. + * + * Named rather than inlined so a column added to either model shows up as a + * diff here rather than as a silent omission from the file. + */ +const DOCUMENT_CONDITION_LINK_BACKUP_SELECT = { + documentId: true, + episodeId: true, + createdAt: true, +} satisfies Prisma.DocumentConditionLinkSelect; + +const EXTRACTED_FACT_BACKUP_SELECT = { + id: true, + documentId: true, + factType: true, + status: true, + confidence: true, + needsReview: true, + committedRecordId: true, + committedRecordType: true, + dataEncrypted: true, + provenanceEncrypted: true, + createdAt: true, + updatedAt: true, +} satisfies Prisma.ExtractedFactSelect; + +/** + * Decrypt one JSON column, or say plainly that this one could not be read. + * + * A single row encrypted under a key the instance has since dropped must not + * take the export down with it: the rest of the vault is still the person's. + * The placeholder is an object rather than `null` because `null` reads as "the + * extraction found nothing here", and that is not what happened. + */ +function decryptJsonSoft(bytes: Uint8Array): unknown { + try { + return JSON.parse(decryptFromBytes(bytes)); + } catch { + return { unreadable: "encrypted with a key this instance no longer holds" }; + } +} + +export async function buildDocumentFilingBackupSection( + prisma: Pick, + userId: string, + options: DocumentFilingBackupOptions = {}, +): Promise { + const disasterRecovery = options.purpose === "disaster-recovery"; + + // Both parents are foreign keys, so a row whose parent the file does not + // carry could only ever restore as a reported loss. The document reader + // skips tombstoned documents in BOTH purposes, and the episode reader skips + // tombstoned episodes in a portable export only — so the filters here mirror + // exactly what `records-backup.ts` decided to carry. + const [linkRows, factRows] = await Promise.all([ + prisma.documentConditionLink.findMany({ + where: { + userId, + document: { deletedAt: null }, + ...(disasterRecovery ? {} : { episode: { deletedAt: null } }), + }, + orderBy: { createdAt: "asc" }, + select: DOCUMENT_CONDITION_LINK_BACKUP_SELECT, + }), + prisma.extractedFact.findMany({ + where: { userId, document: { deletedAt: null } }, + orderBy: { createdAt: "asc" }, + select: EXTRACTED_FACT_BACKUP_SELECT, + }), + ]); + + return { + documentConditionLinks: linkRows.map((row) => ({ + documentId: row.documentId, + episodeId: row.episodeId, + createdAt: row.createdAt.toISOString(), + })), + extractedFacts: factRows.map((row) => ({ + id: row.id, + documentId: row.documentId, + factType: row.factType, + status: row.status, + confidence: row.confidence, + needsReview: row.needsReview, + committedRecordId: row.committedRecordId, + committedRecordType: row.committedRecordType, + ...(disasterRecovery + ? { + dataEncrypted: Buffer.from(row.dataEncrypted).toString("base64"), + provenanceEncrypted: Buffer.from(row.provenanceEncrypted).toString( + "base64", + ), + } + : { + data: decryptJsonSoft(row.dataEncrypted), + provenance: decryptJsonSoft(row.provenanceEncrypted), + }), + createdAt: row.createdAt.toISOString(), + updatedAt: row.updatedAt.toISOString(), + })), + }; +} + +/** Row counts for the audit trail, mirroring the other section counters. */ +export function countDocumentFilingBackupSection( + section: DocumentFilingBackupSection, +): DocumentFilingBackupCounts { + return { + documentConditionLinks: section.documentConditionLinks.length, + extractedFacts: section.extractedFacts.length, + }; +} + +/** Counts this restore wiped, for the audit trail. */ +export interface DocumentFilingRestoreCleared { + documentConditionLinks: number; + extractedFacts: number; +} + +type OptionalNullable = { [K in keyof T]?: T[K] | undefined }; + +/** + * What the parser hands over, which is looser than what the builder writes. + * + * The wire schema defaults or leaves optional every field an older file might + * not carry. The three kept strictly required on a fact are `status`, + * `needsReview` and `confidence`: a file that does not state a review decision + * must not have one invented for it, and making them optional here would let + * exactly that compile. + */ +export type RestoredExtractedFact = Pick< + ExtractedFactBackupEntry, + | "id" + | "documentId" + | "factType" + | "status" + | "confidence" + | "needsReview" + | "createdAt" + | "updatedAt" +> & + OptionalNullable< + Pick< + ExtractedFactBackupEntry, + | "committedRecordId" + | "committedRecordType" + | "dataEncrypted" + | "provenanceEncrypted" + | "data" + | "provenance" + > + >; + +export interface DocumentFilingRestoreInput { + documentConditionLinks: DocumentConditionLinkBackupEntry[]; + extractedFacts: RestoredExtractedFact[]; +} + +/** + * The rows this restore has already written that these two can point at. + * + * Passed in rather than re-queried so this stays a pure function of the + * transaction it was handed, the same arrangement the Coach attachments use. + * `committedRecordIds` is the UNION of the restored lab results, condition + * episodes and medications, because `committedRecordType` chooses between the + * three and a fact naming a row from any of them is equally valid. + */ +export interface DocumentFilingRestoreRefs { + documentIds: ReadonlySet; + episodeIds: ReadonlySet; + committedRecordIds: ReadonlySet; +} + +/** + * Re-create the filings and the staged facts. + * + * Delete-then-recreate inside the caller's transaction, matching every other + * section. Both tables cascade from `InboundDocument`, so the document wipe + * earlier in the restore has already emptied them — they are counted and + * cleared explicitly anyway, so the cleared numbers are truthful rather than + * "whatever the cascade took" and so this stays correct if the document wipe + * ever moves. + * + * MUST be called AFTER the documents, the condition episodes, the lab results + * and the medications are restored. The first two are foreign keys and would + * abort the transaction; the last two are what `committedRecordId` resolves + * against, and resolving before they exist would null every commitment and + * still report success. + */ +export async function restoreDocumentFilingData( + tx: Prisma.TransactionClient, + ownerId: string, + payload: DocumentFilingRestoreInput, + refs: DocumentFilingRestoreRefs, + skips: RestoreSkipLog, +): Promise { + const [clearedLinks, clearedFacts] = await Promise.all([ + tx.documentConditionLink.deleteMany({ where: { userId: ownerId } }), + tx.extractedFact.deleteMany({ where: { userId: ownerId } }), + ]); + + const droppedFilings: string[] = []; + const writableLinks = payload.documentConditionLinks.filter((link) => { + if ( + refs.documentIds.has(link.documentId) && + refs.episodeIds.has(link.episodeId) + ) { + return true; + } + // Reported under the end that is missing, so an operator reading the list + // can tell "the page is gone" from "the condition is gone". + droppedFilings.push( + refs.documentIds.has(link.documentId) ? link.episodeId : link.documentId, + ); + return false; + }); + if (writableLinks.length > 0) { + await tx.documentConditionLink.createMany({ + data: writableLinks.map((link) => ({ + userId: ownerId, + documentId: link.documentId, + episodeId: link.episodeId, + createdAt: new Date(link.createdAt), + })), + }); + } + recordUnknownKeys( + skips, + "documentConditionLink", + [...new Set(droppedFilings)], + droppedFilings, + ); + + const droppedFacts: string[] = []; + const danglingCommitments: string[] = []; + const writableFacts = payload.extractedFacts.filter((fact) => { + if (refs.documentIds.has(fact.documentId)) return true; + droppedFacts.push(fact.documentId); + return false; + }); + if (writableFacts.length > 0) { + await tx.extractedFact.createMany({ + data: writableFacts.map((fact) => { + // Nulled as a PAIR. A type with no id claims the fact was committed + // somewhere and cannot say where, which sends a reader looking for a + // row that is not there. + let committedRecordId = fact.committedRecordId ?? null; + let committedRecordType = fact.committedRecordType ?? null; + if ( + committedRecordId && + !refs.committedRecordIds.has(committedRecordId) + ) { + danglingCommitments.push(committedRecordId); + committedRecordId = null; + committedRecordType = null; + } + return { + id: fact.id, + userId: ownerId, + documentId: fact.documentId, + factType: fact.factType as never, + status: fact.status as never, + confidence: fact.confidence, + needsReview: fact.needsReview, + committedRecordId, + committedRecordType, + dataEncrypted: resolveFactBytes(fact.dataEncrypted, fact.data), + provenanceEncrypted: resolveFactBytes( + fact.provenanceEncrypted, + fact.provenance, + ), + createdAt: new Date(fact.createdAt), + updatedAt: new Date(fact.updatedAt), + }; + }), + }); + } + recordUnknownKeys( + skips, + "extractedFact", + [...new Set(droppedFacts)], + droppedFacts, + ); + recordUnknownKeys( + skips, + "factCommitment", + [...new Set(danglingCommitments)], + danglingCommitments, + ); + + return { + documentConditionLinks: clearedLinks.count, + extractedFacts: clearedFacts.count, + }; +} + +/** + * A fact column's stored bytes, whichever end of the contract the file came + * from. + * + * A disaster-recovery file carries ciphertext that decodes straight back into + * the column. A portable file carries the decrypted JSON, which has to be + * encrypted on the way in under the TARGET instance's key — that is what makes + * a portable file portable. An absent column encrypts `null` rather than + * throwing, because the column is NOT NULL and a fact with unreadable + * provenance is still a fact. + */ +function resolveFactBytes( + ciphertext: string | undefined, + json: unknown, +): Uint8Array { + if (ciphertext !== undefined) { + const decoded = Buffer.from(ciphertext, "base64"); + const bytes = new Uint8Array(new ArrayBuffer(decoded.byteLength)); + bytes.set(decoded); + return bytes; + } + return encryptToBytes(JSON.stringify(json ?? null)); +} diff --git a/src/lib/export/full-backup-payload.ts b/src/lib/export/full-backup-payload.ts index d4ca0176a..8f64432a9 100644 --- a/src/lib/export/full-backup-payload.ts +++ b/src/lib/export/full-backup-payload.ts @@ -72,6 +72,12 @@ import { type CoachMemoryBackupCounts, type CoachMemoryBackupSection, } from "@/lib/export/coach-backup"; +import { + buildDocumentFilingBackupSection, + countDocumentFilingBackupSection, + type DocumentFilingBackupCounts, + type DocumentFilingBackupSection, +} from "@/lib/export/document-filing-backup"; import { buildIntradayProfileBackupSection, countIntradayProfileBackupSection, @@ -90,7 +96,8 @@ export interface FullBackupCounts RemindersBackupCounts, CoachBackupCounts, CoachMemoryBackupCounts, - SensitiveBackupCounts { + SensitiveBackupCounts, + DocumentFilingBackupCounts { measurements: number; medications: number; intakeEvents: number; @@ -317,6 +324,7 @@ export async function buildFullBackupPayload( coach, coachMemory, sensitive, + documentFiling, nutrientDays, ] = await Promise.all([ disasterRecovery @@ -506,6 +514,15 @@ export async function buildFullBackupPayload( buildSensitiveBackupSection(prisma, userId, { purpose: disasterRecovery ? "disaster-recovery" : "portable-export", }), + // Which conditions a document was filed against, and what the extraction + // pass read out of it. Both ends live in + // `src/lib/export/document-filing-backup.ts` beside each other, the same + // arrangement as the sections above and for the same reason. The documents + // themselves ride in `records-backup.ts`; these two say what was done with + // them, which a restored vault otherwise loses in silence. + buildDocumentFilingBackupSection(prisma, userId, { + purpose: disasterRecovery ? "disaster-recovery" : "portable-export", + }), // Nutrient day totals were absent from every export path, which // contradicted the schema's own reason for denormalising the unit column // ("rows stay self-describing in exports even if the catalog ever drifts"). @@ -542,6 +559,7 @@ export async function buildFullBackupPayload( consentReceipts: sensitive.consentReceipts, }; const sensitiveManifest: SensitiveBackupManifest = sensitive.manifest; + const documentFilingSection: DocumentFilingBackupSection = documentFiling; const payload = { schemaVersion: BACKUP_SCHEMA_VERSION, @@ -897,6 +915,7 @@ export async function buildFullBackupPayload( // `manifest` key, and two of them would silently shadow each other // depending on spread order. manifest: { ...recordsSection.manifest, ...sensitiveManifest }, + ...documentFilingSection, nutrientDays: nutrientDays.map((n) => ({ day: n.day, nutrient: n.nutrient, @@ -949,6 +968,7 @@ export async function buildFullBackupPayload( ...countCoachBackupSection(coach), ...countCoachMemoryBackupSection(coachMemory), ...countSensitiveBackupSection(sensitive), + ...countDocumentFilingBackupSection(documentFiling), }, }; } diff --git a/src/lib/export/restore-skips.ts b/src/lib/export/restore-skips.ts index cf7d7dd03..8e4bdeb81 100644 --- a/src/lib/export/restore-skips.ts +++ b/src/lib/export/restore-skips.ts @@ -73,7 +73,25 @@ * reminder still fires. A portable export omits tombstoned plans, which is the * ordinary way a live reminder ends up naming one the file does not carry. * - * The tenth, `checkupClosure`, is not about a restore at all, and it borrows + * The tenth and eleventh, `documentConditionLink` and `extractedFact`, name a + * vault row whose parent the restore did not put back — the page a filing was + * made against, the condition it was filed under, or the document a staged + * fact was read out of. All three are real foreign keys, so unlike every kind + * above the alternative to dropping the row is not a dangling pointer but an + * aborted transaction and no restore at all. The builder carries a filing or a + * fact only when both of its ends are carried, so a file this release writes + * never trips either — they exist for the hand-edited or truncated file. + * + * The twelfth, `factCommitment`, is the pointer kind rather than the row kind, + * exactly like `coachReference`: `ExtractedFact.committedRecordId` names the + * lab result, condition episode or medication an approved fact was committed + * to, and it is a bare id column with no relation, so a value pointing at + * nothing costs no error and simply stops meaning anything. The fact itself + * restores; what it loses is a pointer that was already going nowhere, and its + * `committedRecordType` goes with it so the row does not claim a commitment it + * cannot name. + * + * The thirteenth, `checkupClosure`, is not about a restore at all, and it borrows * this shape deliberately rather than growing a second reporting mechanism * beside it. The situation is the same one: something a write was asked to do * could not be done, the record itself survives, and the person is told which @@ -91,6 +109,9 @@ export type SkippedCatalogue = | "reminderReference" | "coachAttachment" | "coachReference" + | "documentConditionLink" + | "extractedFact" + | "factCommitment" | "checkupClosure"; /** One key this instance does not know, and the links it cost. */ diff --git a/src/lib/links/__tests__/link-surface-guard.test.ts b/src/lib/links/__tests__/link-surface-guard.test.ts index 41e17c9ca..086af0bfe 100644 --- a/src/lib/links/__tests__/link-surface-guard.test.ts +++ b/src/lib/links/__tests__/link-surface-guard.test.ts @@ -58,15 +58,20 @@ const LINK_CALL_RE = new RegExp( const MODULE_DIR = "lib/links/"; /** - * The two files exempt by the same standing rule the wipe and key-rotation - * writers hold: a backup RESTORE rebuilds an account rather than filing - * something in one, so it preserves ids and creation instants and writes the - * link tables directly. It is the only exemption, and it is frozen here so a - * third arrives as a diff a human reviewed rather than as a quiet spread. + * The files exempt by the same standing rule the wipe and key-rotation writers + * hold: a backup RESTORE rebuilds an account rather than filing something in + * one, so it preserves ids and creation instants and writes the link tables + * directly. The set is frozen here so the next one arrives as a diff a human + * reviewed rather than as a quiet spread. */ const RESTORE_EXEMPTIONS = [ "lib/export/visits-backup.ts", "lib/export/vaccinations-backup.ts", + // The document↔condition filing, carried and restored from the release that + // takes it off the backup debt register. Same standing rule as the two + // above: it re-creates the link with the instant it was originally filed at, + // which the service's filing signature has no way to express. + "lib/export/document-filing-backup.ts", ].sort(); /** Every non-test, non-generated source file under `src/`, relative to `src/`. */ @@ -123,14 +128,15 @@ describe("link-surface guard — the tables have one gateway", () => { } }); - it("the restore exemption set is exactly the two backup writers", () => { + it("the restore exemption set is exactly the three backup writers", () => { // Freeze the exemption list so widening it is a reviewed diff, not a quiet - // addition. Both files must exist and must actually write a link table. + // addition. Every file must exist and must actually write a link table. for (const rel of RESTORE_EXEMPTIONS) { expect(matchedDelegates(read(rel)).length).toBeGreaterThan(0); } expect(RESTORE_EXEMPTIONS).toEqual( [ + "lib/export/document-filing-backup.ts", "lib/export/vaccinations-backup.ts", "lib/export/visits-backup.ts", ].sort(), diff --git a/src/lib/validations/backup.ts b/src/lib/validations/backup.ts index a02109f0c..2c9488e6e 100644 --- a/src/lib/validations/backup.ts +++ b/src/lib/validations/backup.ts @@ -34,6 +34,8 @@ import { DocumentSummaryState, EncounterKind, EncounterStatus, + ExtractedFactStatus, + ExtractedFactType, FamilyRelationship, FlowLevel, GlucoseContext, @@ -1215,6 +1217,49 @@ const coachReminderBackupSchema = z }) .passthrough(); +/** One document filed against one condition episode. */ +const documentConditionLinkBackupSchema = z + .object({ + documentId: z.string().min(1), + episodeId: z.string().min(1), + createdAt: isoDateTime, + }) + .passthrough(); + +/** + * One fact the extraction pass staged against a document. + * + * `status`, `confidence` and `needsReview` are REQUIRED rather than defaulted, + * unlike most optional-looking fields in this file. Each has a schema default + * that means "nobody has looked at this yet", and the confirm endpoint acts on + * a PENDING fact by committing it into the structured store — so a file that + * does not state the review decision must fail to parse rather than have one + * invented for it and let an already-committed reading be approved twice. + * + * `dataEncrypted` / `data` and their provenance siblings are the two ends of + * the same contract, so all four are optional and exactly one pair arrives: a + * disaster-recovery file carries the ciphertext, a portable file carries the + * decrypted JSON. + */ +const extractedFactBackupSchema = z + .object({ + id: z.string().min(1), + documentId: z.string().min(1), + factType: z.enum(ExtractedFactType), + status: z.enum(ExtractedFactStatus), + confidence: z.number(), + needsReview: z.boolean(), + committedRecordId: z.string().nullable().optional(), + committedRecordType: z.string().nullable().optional(), + dataEncrypted: base64BytesSchema.optional(), + provenanceEncrypted: base64BytesSchema.optional(), + data: z.json().optional(), + provenance: z.json().optional(), + createdAt: isoDateTime, + updatedAt: isoDateTime, + }) + .passthrough(); + /** * A grouping the account created for its own mood factors. `id` is REQUIRED, * unlike everywhere else in this file, because a custom tag addresses its @@ -1414,6 +1459,13 @@ export const backupPayloadSchema = z familyHistory: z.array(familyHistoryBackupSchema).default([]), workouts: z.array(workoutBackupSchema).default([]), documents: z.array(documentBackupSchema).default([]), + // What a document was filed against, and what was read out of it. Defaulted + // for the same reason as the sections below: a file written before the + // vault filing travelled carries no key, and an unsorted vault writes []. + documentConditionLinks: z + .array(documentConditionLinkBackupSchema) + .default([]), + extractedFacts: z.array(extractedFactBackupSchema).default([]), nutrientDays: z.array(nutrientDaySchema).default([]), // Durable self-context and user-defined series. Defaulted so files written // before either rode the wire still parse; an account with neither writes @@ -1518,6 +1570,10 @@ export interface BackupSummary { workouts: number; /** Document records (ciphertext included in canonical DR payloads). */ documents: number; + /** Which documents were filed against which condition. */ + documentConditionLinks: number; + /** Facts staged out of a document, with their review decision. */ + extractedFacts: number; /** 1 when the account's durable self-context rides the file, 0 otherwise. */ healthProfile: number; /** Effective-dated structured health-profile revisions. */ @@ -1587,6 +1643,11 @@ export function summarizeBackup(payload: BackupPayload): BackupSummary { familyHistory: payload.familyHistory.length, workouts: payload.workouts.length, documents: payload.documents.length, + // Counted from the release that carries them, so the admin's "what did I + // just restore" answer never under-counts a filed vault the way it once + // did for visits. + documentConditionLinks: payload.documentConditionLinks.length, + extractedFacts: payload.extractedFacts.length, healthProfile: payload.healthProfile ? 1 : 0, healthProfileFactRevisions: payload.healthProfileFacts.length, customMetrics: payload.customMetrics.length, diff --git a/tests/integration/backup-round-trip.test.ts b/tests/integration/backup-round-trip.test.ts index 36fa1c6e9..4c5b8d607 100644 --- a/tests/integration/backup-round-trip.test.ts +++ b/tests/integration/backup-round-trip.test.ts @@ -26,18 +26,22 @@ * The registry below is keyed by `TwoEndedModel`, so a name added to the plan's * two-ended list without a row seeded and counted here does not compile. * - * What this still does not prove, with one exception: that a restored row - * carries the right VALUES. It asks whether the rows came back, not whether - * they came back intact — `admin-backups-canonical-roundtrip.test.ts` is where - * field-level fidelity is asserted. A restore that wrote one row per model with - * every column defaulted would satisfy this file and fail that one. + * What the count-back alone does not prove: that a restored row carries the + * right VALUES. It asks whether the rows came back, not whether they came back + * intact — `admin-backups-canonical-roundtrip.test.ts` is where field-level + * fidelity is asserted for the file as a whole. A restore that wrote one row + * per model with every column defaulted would satisfy the counts and fail that + * one. * - * The exception is the medication side effect. Its note lives in an encrypted - * column beside a legacy plaintext one, so "the row came back" and "the note - * came back" are genuinely different answers here: a restore that dropped the - * ciphertext would still be counted as recovered by everything above. The - * severity, the category and the entry are asserted alongside it, because a - * side effect without them says something happened and not what. + * So each section added since has brought a field-level assertion of its own + * for the column whose loss the count cannot see: the side effect's encrypted + * note beside its legacy plaintext one, the open pause era's null `resumedAt`, + * the Coach's permanent fence flag and its two bare-id references, the stock + * count that must not be recalculated from its ledger, the dose ramp's order, + * the reminder's snooze and skip cursors — and, at the bottom of this file, the + * document filing's PAIRING and the review decision on a staged fact. Each one + * is a thing a restore could get wrong while returning the right number of + * rows, which is the only test worth writing here. */ import { beforeEach, describe, expect, it, vi } from "vitest"; @@ -48,6 +52,12 @@ import type { PrismaClient } from "@/generated/prisma/client"; import { encrypt, encryptBytes } from "@/lib/crypto"; import { decryptFromBytes, encryptToBytes } from "@/lib/ai/coach/bytes-codec"; import { readNote } from "@/lib/crypto/note-cipher"; +import { + decryptFactData, + decryptFactProvenance, + encryptFactData, + encryptFactProvenance, +} from "@/lib/documents/store"; import { buildFullBackupPayload } from "@/lib/export/full-backup-payload"; import { TWO_ENDED_MODELS, type TwoEndedModel } from "@/lib/export/backup-plan"; import { POST } from "@/app/api/admin/backups/[id]/restore/route"; @@ -97,6 +107,7 @@ const COACH_USER_TURN = "my readings look higher this week, is that real?"; const COACH_ASSISTANT_TURN = "the last seven mornings average 4 mmHg above the fortnight before"; const DOSE_CHANGE_NOTE = "titration note, encrypted at rest"; +const EXTRACTED_FACT_SPAN = "Ferritin 91 ng/mL (30 - 400)"; const SIDE_EFFECT_NOTE = "nausea for two hours after the evening dose"; beforeEach(async () => { @@ -201,6 +212,9 @@ const COUNT_BACK: Record< MentalHealthAssessment: (p, userId) => p.mentalHealthAssessment.count({ where: { userId } }), ConsentReceipt: (p, userId) => p.consentReceipt.count({ where: { userId } }), + DocumentConditionLink: (p, userId) => + p.documentConditionLink.count({ where: { userId } }), + ExtractedFact: (p, userId) => p.extractedFact.count({ where: { userId } }), }; /** @@ -508,6 +522,18 @@ async function seedEveryTwoEndedModel(prisma: PrismaClient): Promise { onsetAt: AT("2026-06-20T00:00:00.000Z"), }, }); + // A SECOND condition, and the one the document below is actually filed + // against. Two episodes is what makes the filing assertion mean something: + // with one, a restore that pointed every link at the first condition it + // found would return the right count and read as correct. + const filedEpisode = await prisma.illnessEpisode.create({ + data: { + userId: OWNER_ID, + label: "Iron deficiency", + type: "CHRONIC", + onsetAt: AT("2026-03-01T00:00:00.000Z"), + }, + }); const illnessSymptom = await prisma.illnessSymptom.create({ data: { key: "round_trip_cough", labelKey: "illness.symptom.roundTrip" }, }); @@ -643,6 +669,56 @@ async function seedEveryTwoEndedModel(prisma: PrismaClient): Promise { }, }); + // The page filed under the SECOND condition, not the first. A restore that + // kept the count and lost the pairing would leave the vault sorted wrongly + // rather than visibly unsorted, which is harder to notice and worse. + await prisma.documentConditionLink.create({ + data: { + userId: OWNER_ID, + documentId: document.id, + episodeId: filedEpisode.id, + createdAt: AT("2026-07-02T09:00:00.000Z"), + }, + }); + + // One staged fact, already reviewed, approved and committed to the ferritin + // lab row above. Every column that records the decision is set AWAY from its + // schema default on purpose: `PENDING`, `needsReview: true` and two NULL + // commitment columns are what a restore that ignores them writes, and the + // assertion after the restore is what catches that. A fact handed back as + // PENDING is offered for review again, and approving it a second time writes + // a second lab result for a reading the account already has. + await prisma.extractedFact.create({ + data: { + userId: OWNER_ID, + documentId: document.id, + factType: "OBSERVATION", + status: "APPROVED", + confidence: 0.94, + needsReview: false, + committedRecordId: labResult.id, + committedRecordType: "labResult", + dataEncrypted: encryptFactData({ + label: "Ferritin", + code: null, + codeSystem: null, + value: 91, + valueText: null, + unit: "ng/mL", + referenceLow: 30, + referenceHigh: 400, + effectiveDate: "2026-06-30", + }), + provenanceEncrypted: encryptFactProvenance({ + sourceText: EXTRACTED_FACT_SPAN, + anchored: true, + sourceOffset: 412, + page: 2, + confidence: 0.94, + }), + }, + }); + // Two Coach threads, and the pair is the point. One is an ordinary health // conversation; the other is FENCED — `documentScoped` true, grounded in the // document seeded above. A restore that let the flag default to false would @@ -1457,6 +1533,95 @@ describe("every model the plan claims two-ended survives a real restore", () => }, ]); + // The vault came back SORTED, not merely populated. + // + // The count says one filing returned. It cannot say which page was filed + // under which condition, and the account has two conditions — so a restore + // that paired the document with the first episode it found would satisfy + // every count above and hand back a lab report filed under a head cold. + // The pair is resolved through the relations rather than compared against + // the ids the fixture used, because both ends were re-created by the + // restore and it is the pairing that has to survive, not the identifiers. + const filing = await prisma.documentConditionLink.findFirstOrThrow({ + where: { userId: OWNER_ID }, + include: { + document: { select: { title: true } }, + episode: { select: { label: true } }, + }, + }); + expect( + { + document: filing.document.title, + condition: filing.episode.label, + createdAt: filing.createdAt.toISOString(), + }, + "the page must come back filed under the condition it was filed under", + ).toEqual({ + document: "June labs", + condition: "Iron deficiency", + createdAt: "2026-07-02T09:00:00.000Z", + }); + + // The staged fact, and the decision on it. + // + // This is the assertion the count cannot make. `status`, `needsReview`, + // `committedRecordId` and `committedRecordType` all default to "nobody has + // looked at this yet", so a restore that wrote the row and ignored them + // returns exactly one fact, exactly as the plan promises, and hands the + // account a reviewed and committed reading back in its review queue. The + // confirm endpoint acts only on a PENDING fact and commits it through the + // normal create, so approving it a second time writes a SECOND ferritin + // result. The commitment is checked against the lab row the restore itself + // wrote, because a pointer that survives as a string but no longer names a + // live row is the same loss wearing a value. + const restoredLab = await prisma.labResult.findFirstOrThrow({ + where: { userId: OWNER_ID }, + }); + const stagedFact = await prisma.extractedFact.findFirstOrThrow({ + where: { userId: OWNER_ID }, + include: { document: { select: { title: true } } }, + }); + expect( + { + document: stagedFact.document.title, + factType: stagedFact.factType, + status: stagedFact.status, + confidence: stagedFact.confidence, + needsReview: stagedFact.needsReview, + committedRecordId: stagedFact.committedRecordId, + committedRecordType: stagedFact.committedRecordType, + data: decryptFactData(stagedFact.dataEncrypted), + provenance: decryptFactProvenance(stagedFact.provenanceEncrypted), + }, + "a reviewed and committed fact must not come back up for review", + ).toEqual({ + document: "June labs", + factType: "OBSERVATION", + status: "APPROVED", + confidence: 0.94, + needsReview: false, + committedRecordId: restoredLab.id, + committedRecordType: "labResult", + data: { + label: "Ferritin", + code: null, + codeSystem: null, + value: 91, + valueText: null, + unit: "ng/mL", + referenceLow: 30, + referenceHigh: 400, + effectiveDate: "2026-06-30", + }, + provenance: { + sourceText: EXTRACTED_FACT_SPAN, + anchored: true, + sourceOffset: 412, + page: 2, + confidence: 0.94, + }, + }); + // The appointment's reminder reference survives too — same remap, other // referrer. const restoredEncounter = await prisma.encounter.findFirstOrThrow({ @@ -1582,4 +1747,190 @@ describe("every model the plan claims two-ended survives a real restore", () => reminderId: null, }); }); + + /** + * The hand-edited file, which is the only way the vault filing can carry a + * reference the restore cannot place. + * + * The builder carries a filing or a staged fact only when both of its ends + * are carried, so no file this release writes reaches the drop path. That is + * exactly why the path is worth a test of its own: an arm nothing exercises + * is an arm nobody notices is wrong, and here being wrong has two very + * different prices. `documentConditionLink.episodeId` is a real foreign key, + * so writing it unchecked does not lose one edge — it violates a constraint + * and rolls the WHOLE account back, which is what the mood categories did + * before they travelled. `ExtractedFact.committedRecordId` is a bare id + * column with no relation, so writing it unchecked costs no error at all and + * simply leaves a fact pointing at a lab result that is not there. + * + * Both references are broken in the same file so the two answers can be seen + * side by side: a dropped row and a nulled pointer, each named in the report, + * and a restore that still answers 200. + */ + it("drops and names a filing and a commitment a truncated file cannot resolve", async () => { + const prisma = getPrismaClient(); + await seedAdminSession(prisma); + await createOwner(prisma); + + const episode = await prisma.illnessEpisode.create({ + data: { + userId: OWNER_ID, + label: "Iron deficiency", + type: "CHRONIC", + onsetAt: AT("2026-03-01T00:00:00.000Z"), + }, + }); + const labResult = await prisma.labResult.create({ + data: { + userId: OWNER_ID, + analyte: "Ferritin", + value: 91, + unit: "ng/mL", + takenAt: AT("2026-06-30T09:00:00.000Z"), + }, + }); + const documentBytes = encryptBytes(Buffer.from("truncated-file fixture")); + const contentEncrypted = new Uint8Array( + new ArrayBuffer(documentBytes.byteLength), + ); + contentEncrypted.set(documentBytes); + const document = await prisma.inboundDocument.create({ + data: { + userId: OWNER_ID, + kind: "LAB_RESULT", + title: "June labs", + mimeType: "application/pdf", + byteSize: documentBytes.byteLength, + contentEncrypted, + contentCodec: "binary2", + }, + }); + await prisma.documentConditionLink.create({ + data: { + userId: OWNER_ID, + documentId: document.id, + episodeId: episode.id, + }, + }); + await prisma.extractedFact.create({ + data: { + userId: OWNER_ID, + documentId: document.id, + factType: "OBSERVATION", + status: "APPROVED", + confidence: 0.9, + needsReview: false, + committedRecordId: labResult.id, + committedRecordType: "labResult", + dataEncrypted: encryptFactData({ + label: "Ferritin", + code: null, + codeSystem: null, + value: 91, + valueText: null, + unit: "ng/mL", + referenceLow: null, + referenceHigh: null, + effectiveDate: "2026-06-30", + }), + provenanceEncrypted: encryptFactProvenance({ + sourceText: EXTRACTED_FACT_SPAN, + anchored: true, + sourceOffset: 412, + page: 2, + confidence: 0.9, + }), + }, + }); + + const { payload } = await buildFullBackupPayload(prisma, OWNER_ID, { + purpose: "disaster-recovery", + }); + // The edit a truncated or hand-repaired file makes: both references now + // name rows the file no longer carries. + const edited = payload as typeof payload & { + documentConditionLinks: Array<{ episodeId: string }>; + extractedFacts: Array<{ committedRecordId: string | null }>; + }; + edited.documentConditionLinks[0].episodeId = "condition-not-in-this-file"; + edited.extractedFacts[0].committedRecordId = "lab-not-in-this-file"; + + await prisma.user.delete({ where: { id: OWNER_ID } }); + await createOwner(prisma); + + const backup = await prisma.dataBackup.create({ + data: { + userId: OWNER_ID, + type: "TWO_ENDED_ROUND_TRIP", + data: encrypt(JSON.stringify(payload)), + }, + }); + const response = await POST( + new Request(`http://localhost/api/admin/backups/${backup.id}/restore`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ confirm: "RESTORE" }), + }) as never, + { params: Promise.resolve({ id: backup.id }) }, + ); + const body = await response.json(); + expect( + response.status, + "a filing naming a condition the file does not carry must cost that " + + "filing, not the account", + ).toBe(200); + + const reported = body.data.skipped.catalogueKeys as Array<{ + catalogue: string; + key: string; + links: number; + }>; + expect( + reported.filter((entry) => + ["documentConditionLink", "factCommitment"].includes(entry.catalogue), + ), + ).toEqual([ + { + catalogue: "documentConditionLink", + key: "condition-not-in-this-file", + links: 1, + }, + { catalogue: "factCommitment", key: "lab-not-in-this-file", links: 1 }, + ]); + + // The filing is gone because it had nowhere to hang. The fact is NOT: what + // it lost is a pointer that was already going nowhere, and the transcribed + // reading is still the account's. + expect( + await prisma.documentConditionLink.count({ where: { userId: OWNER_ID } }), + ).toBe(0); + const survivor = await prisma.extractedFact.findFirstOrThrow({ + where: { userId: OWNER_ID }, + }); + expect({ + status: survivor.status, + needsReview: survivor.needsReview, + committedRecordId: survivor.committedRecordId, + committedRecordType: survivor.committedRecordType, + data: decryptFactData(survivor.dataEncrypted), + }).toEqual({ + status: "APPROVED", + needsReview: false, + // Nulled as a pair: a type with no id would claim a commitment the row + // cannot name. + committedRecordId: null, + committedRecordType: null, + data: { + label: "Ferritin", + code: null, + codeSystem: null, + value: 91, + valueText: null, + unit: "ng/mL", + referenceLow: null, + referenceHigh: null, + effectiveDate: "2026-06-30", + }, + }); + }); }); From 352ff617da399045a71950ab19af99df84f614a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 22 Aug 2026 10:42:32 +0200 Subject: [PATCH 2/2] Stop the switched-payload spec failing on its own reload The scoped-sharing spec routes `/api/auth/me`, and the account switch it drives ends in a full reload. The reload cancels whatever call is in flight, Playwright marks the route handled, and the handler's `fulfill` then lands on it with "Route is already handled!". With `failOnFlakyTests` on CI that turns the whole shard red over a teardown race rather than over anything the test is checking. Same guard the record-session-fence spec and this file's own accessibility sibling already use. The `fetch` is guarded too, because a request cancelled mid-flight rejects there rather than at the fulfill, and the earlier of the two is the one that decides which error the run reports. --- e2e/v137-sharing-managed-profiles.spec.ts | 29 ++++++++++++++++++----- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/e2e/v137-sharing-managed-profiles.spec.ts b/e2e/v137-sharing-managed-profiles.spec.ts index b170c49fe..7799d023f 100644 --- a/e2e/v137-sharing-managed-profiles.spec.ts +++ b/e2e/v137-sharing-managed-profiles.spec.ts @@ -291,15 +291,32 @@ test.describe.serial("scoped sharing browser journeys", () => { let corruptAuthPayload = true; await page.route("**/api/auth/me", async (route) => { - const response = await route.fetch(); + // The switch this test drives ends in a full reload, and the reload + // cancels whatever `/api/auth/me` call is in flight. Playwright then + // considers the route handled, and the `fulfill` below lands on it with + // "Route is already handled!" — a failure of the teardown, not of the + // thing under test, and one that turns the whole shard red under + // `failOnFlakyTests`. + // + // Same guard as `v137-record-session-fence.spec.ts` and the a11y sibling + // of this file. The fetch is guarded too: a request cancelled mid-flight + // rejects there rather than at the fulfill. + let response; + try { + response = await route.fetch(); + } catch { + return; + } if (!corruptAuthPayload) { - await route.fulfill({ response }); + await route.fulfill({ response }).catch(() => {}); return; } - await route.fulfill({ - response, - json: withDivergentActiveAccountAccess(await response.json()), - }); + await route + .fulfill({ + response, + json: withDivergentActiveAccountAccess(await response.json()), + }) + .catch(() => {}); }); const ownerOnlyReads: string[] = [];