From 22e51ee4d1391dc3ceffa78698b763b943c4567b Mon Sep 17 00:00:00 2001 From: SoulSpace Creator Date: Tue, 28 Jul 2026 14:56:34 +0800 Subject: [PATCH] feat(knowledge-corpus): implement repository adapter --- .gitignore | 1 + ARCHITECTURE_DECISIONS.md | 8 + CHANGELOG.md | 1 + DOCUMENTATION_INDEX.md | 10 + README.md | 8 +- ...e_Corpus_Candidate_Source_Contract_v1.0.md | 27 ++ ...and_Change_Detection_Specification_v1.0.md | 28 ++ ..._Repository_Snapshot_Specification_v1.0.md | 26 ++ ...S_Milestone_07_Acceptance_Criteria_v1.0.md | 29 ++ ...ilestone_07_Codex_Execution_Prompt_v1.0.md | 87 +++++ ...s_Repository_Adapter_Specification_v1.0.md | 53 +++ ...ilestone_07_Verification_Checklist_v1.0.md | 31 ++ packages/knowledge-schema/README.md | 2 + packages/knowledge-schema/src/corpus.ts | 247 ++++++++++++++ packages/knowledge-schema/src/index.ts | 1 + packages/knowledge-schema/src/migration.ts | 11 +- packages/knowledge-schema/src/parse.ts | 32 ++ packages/knowledge-schema/src/primitives.ts | 3 + .../knowledge-schema/tests/corpus.test.ts | 200 ++++++++++++ services/knowledge-engine/README.md | 18 +- .../compare-knowledge-repository-snapshots.ts | 132 ++++++++ .../initialize-corpus-knowledge-repository.ts | 22 ++ services/knowledge-engine/src/index.ts | 3 + .../knowledge-corpus-candidate-source.ts | 198 ++++++++++++ .../knowledge-corpus-candidate-source.test.ts | 305 ++++++++++++++++++ 25 files changed, 1473 insertions(+), 10 deletions(-) create mode 100644 docs/milestones/milestone-07/FounderOS_Knowledge_Corpus_Candidate_Source_Contract_v1.0.md create mode 100644 docs/milestones/milestone-07/FounderOS_Knowledge_Corpus_Refresh_and_Change_Detection_Specification_v1.0.md create mode 100644 docs/milestones/milestone-07/FounderOS_Knowledge_Repository_Snapshot_Specification_v1.0.md create mode 100644 docs/milestones/milestone-07/FounderOS_Milestone_07_Acceptance_Criteria_v1.0.md create mode 100644 docs/milestones/milestone-07/FounderOS_Milestone_07_Codex_Execution_Prompt_v1.0.md create mode 100644 docs/milestones/milestone-07/FounderOS_Milestone_07_Knowledge_Corpus_Repository_Adapter_Specification_v1.0.md create mode 100644 docs/milestones/milestone-07/FounderOS_Milestone_07_Verification_Checklist_v1.0.md create mode 100644 packages/knowledge-schema/src/corpus.ts create mode 100644 packages/knowledge-schema/tests/corpus.test.ts create mode 100644 services/knowledge-engine/src/application/compare-knowledge-repository-snapshots.ts create mode 100644 services/knowledge-engine/src/application/initialize-corpus-knowledge-repository.ts create mode 100644 services/knowledge-engine/src/infrastructure/knowledge-corpus-candidate-source.ts create mode 100644 services/knowledge-engine/tests/knowledge-corpus-candidate-source.test.ts diff --git a/.gitignore b/.gitignore index dc86ab5..53fd8e5 100644 --- a/.gitignore +++ b/.gitignore @@ -6,5 +6,6 @@ coverage/ dist/ node_modules/ +.pnpm-store/ *.log migration-report.json diff --git a/ARCHITECTURE_DECISIONS.md b/ARCHITECTURE_DECISIONS.md index 273acfc..bb7cfad 100644 --- a/ARCHITECTURE_DECISIONS.md +++ b/ARCHITECTURE_DECISIONS.md @@ -82,6 +82,14 @@ This ledger records repository-level decisions. Feature-level decisions should m - **Decision:** Define versioned candidate-source batches and asynchronous repository interfaces in `@founderos/knowledge-schema`. Implement a validated in-memory candidate source and immutable repository snapshot in `@founderos/knowledge-engine`. Candidate sources provide objects and source provenance; repositories revalidate, reject duplicate identities, sort observable results, and supply candidates to the existing query filter through a repository-backed application service. - **Consequences:** Query execution no longer needs to know how candidates were obtained, and future providers can implement the same asynchronous contract. The in-memory repository is rebuilt from its sources, carries no durability or update semantics, and deliberately performs no ranking, semantic selection, or authorization. +## ADR-0011: Materialize approved corpus access as immutable content-addressed snapshots + +- **Status:** Accepted +- **Date:** 2026-07-28 +- **Context:** Milestone 06 proves repository-backed querying with manually supplied candidates, but future context assembly needs to identify exactly which approved corpus state supplied a result without coupling queries to files or introducing persistence. +- **Decision:** Implement an engine-owned corpus candidate source that delegates canonical reads, approval gates, path safety, source hashes, and object validation to the Milestone 04 migration workflow. Materialize its accepted objects through the existing in-memory repository and create a schema-validated, deeply immutable snapshot whose identity is derived from corpus version, manifest reference, and deterministic per-object fingerprints. Compare snapshots through a pure, sorted change-set contract. +- **Consequences:** The Priority 1 corpus can be queried through the existing repository abstraction with traceable, reproducible knowledge-state identity. Creation metadata does not alter content identity. Change detection is observable but inert: durable storage, automatic refresh, watchers, synchronization, retrieval intelligence, and agent integration remain future decisions. + ## ADR template ```markdown diff --git a/CHANGELOG.md b/CHANGELOG.md index 8001e58..d29e12a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,3 +22,4 @@ All notable changes to FounderOS will be documented here. - Deterministic in-memory query execution and Priority 1 corpus evaluation fixtures. - Versioned candidate-source and Knowledge Repository contracts. - Validated in-memory candidate provider, deterministic repository access, and repository-backed query execution. +- Approved-corpus candidate source, immutable repository snapshots, corpus-backed repository initialization, and deterministic change detection. diff --git a/DOCUMENTATION_INDEX.md b/DOCUMENTATION_INDEX.md index 0fc4751..b9e1180 100644 --- a/DOCUMENTATION_INDEX.md +++ b/DOCUMENTATION_INDEX.md @@ -82,6 +82,16 @@ The documents below are the official FounderOS v1.0 bootstrap specification, org - [Milestone 06 Verification Checklist v1.0](./docs/milestones/milestone-06/FounderOS_Milestone_06_Verification_Checklist_v1.0.md) - [Milestone 06 Codex Execution Prompt v1.0](./docs/milestones/milestone-06/FounderOS_Milestone_06_Codex_Execution_Prompt_v1.0.md) +### Milestone 07 — Knowledge Corpus Repository Adapter + +- [Knowledge Corpus Repository Adapter Specification v1.0](./docs/milestones/milestone-07/FounderOS_Milestone_07_Knowledge_Corpus_Repository_Adapter_Specification_v1.0.md) +- [Knowledge Corpus Candidate Source Contract v1.0](./docs/milestones/milestone-07/FounderOS_Knowledge_Corpus_Candidate_Source_Contract_v1.0.md) +- [Knowledge Repository Snapshot Specification v1.0](./docs/milestones/milestone-07/FounderOS_Knowledge_Repository_Snapshot_Specification_v1.0.md) +- [Knowledge Corpus Refresh and Change Detection Specification v1.0](./docs/milestones/milestone-07/FounderOS_Knowledge_Corpus_Refresh_and_Change_Detection_Specification_v1.0.md) +- [Milestone 07 Acceptance Criteria v1.0](./docs/milestones/milestone-07/FounderOS_Milestone_07_Acceptance_Criteria_v1.0.md) +- [Milestone 07 Verification Checklist v1.0](./docs/milestones/milestone-07/FounderOS_Milestone_07_Verification_Checklist_v1.0.md) +- [Milestone 07 Codex Execution Prompt v1.0](./docs/milestones/milestone-07/FounderOS_Milestone_07_Codex_Execution_Prompt_v1.0.md) + ## Repository governance - [Architecture decisions](./ARCHITECTURE_DECISIONS.md) diff --git a/README.md b/README.md index a59d1cb..a39c869 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ FounderOS is an AI-native operating system for founder decision-making, organizational memory, and governed AI-assisted execution. This repository is a documentation-first TypeScript monorepo. -The repository currently provides the governed KnowledgeOS schema, ingestion, migration, repository access, and deterministic query foundations. It does **not** implement persistence, semantic retrieval, Hermes, an agent runtime, MCP connectors, or a user interface. +The repository currently provides the governed KnowledgeOS schema, ingestion, migration, corpus-backed repository snapshots, change detection, and deterministic query foundations. It does **not** implement persistence, semantic retrieval, Hermes, an agent runtime, MCP connectors, or a user interface. ## Architecture at a glance @@ -20,12 +20,12 @@ The official specifications are indexed in [DOCUMENTATION_INDEX.md](./DOCUMENTAT ## Implemented foundations -- [`@founderos/knowledge-schema`](./packages/knowledge-schema/README.md) provides strict runtime schemas and inferred TypeScript contracts for KnowledgeOS metadata, objects, migration, queries, candidate sources, repositories, and results. -- [`@founderos/knowledge-engine`](./services/knowledge-engine/README.md) provides read-only ingestion, manifest-controlled Priority 1 corpus migration, validated in-memory repository access, and deterministic filtering with preserved source provenance. +- [`@founderos/knowledge-schema`](./packages/knowledge-schema/README.md) provides strict runtime schemas and inferred TypeScript contracts for KnowledgeOS metadata, objects, migration, queries, candidate sources, repository snapshots, change sets, and results. +- [`@founderos/knowledge-engine`](./services/knowledge-engine/README.md) provides read-only ingestion, manifest-controlled Priority 1 corpus migration, corpus-backed repository initialization, deterministic snapshots and change comparison, and exact filtering with preserved source provenance. - [`specs/knowledge-templates`](./specs/knowledge-templates) provides valid Markdown templates for all seven KnowledgeOS object types. - [`knowledge/migration-manifest.yaml`](./knowledge/migration-manifest.yaml) binds the eight canonical FounderOS Priority 1 documents to reviewed object identities, logical destinations, metadata, and source hashes. -Vault watching, durable persistence, semantic retrieval, embeddings, ranking, graph storage, agent behavior, connectors, and interfaces remain unimplemented. +Corpus refresh execution, vault watching, durable persistence, semantic retrieval, embeddings, ranking, graph storage, agent behavior, connectors, and interfaces remain unimplemented. ## Repository layout diff --git a/docs/milestones/milestone-07/FounderOS_Knowledge_Corpus_Candidate_Source_Contract_v1.0.md b/docs/milestones/milestone-07/FounderOS_Knowledge_Corpus_Candidate_Source_Contract_v1.0.md new file mode 100644 index 0000000..aa63e66 --- /dev/null +++ b/docs/milestones/milestone-07/FounderOS_Knowledge_Corpus_Candidate_Source_Contract_v1.0.md @@ -0,0 +1,27 @@ +# FounderOS Knowledge Corpus Candidate Source Contract v1.0 + +## Purpose + +Define the contract for a corpus-backed candidate source. + +## Responsibilities + +The source provides: + +- Knowledge object discovery +- Corpus metadata +- Snapshot identity +- Provenance information + +## Requirements + +The source must: + +- Load approved corpus content +- Preserve object identity +- Preserve provenance +- Maintain deterministic ordering + +## Principle + +A candidate source provides trusted knowledge candidates, not intelligence. diff --git a/docs/milestones/milestone-07/FounderOS_Knowledge_Corpus_Refresh_and_Change_Detection_Specification_v1.0.md b/docs/milestones/milestone-07/FounderOS_Knowledge_Corpus_Refresh_and_Change_Detection_Specification_v1.0.md new file mode 100644 index 0000000..7e158c0 --- /dev/null +++ b/docs/milestones/milestone-07/FounderOS_Knowledge_Corpus_Refresh_and_Change_Detection_Specification_v1.0.md @@ -0,0 +1,28 @@ +# FounderOS Knowledge Corpus Refresh and Change Detection Specification v1.0 + +## Purpose + +Define the foundation for detecting controlled knowledge corpus changes. + +## Detection Inputs + +Compare: + +- Source hashes +- Object identities +- Metadata changes +- Corpus version + +## Scope + +This milestone defines contracts only. + +Deferred: + +- Automatic synchronization +- Background refresh +- Event streaming + +## Principle + +Knowledge changes must be observable before they become operational. diff --git a/docs/milestones/milestone-07/FounderOS_Knowledge_Repository_Snapshot_Specification_v1.0.md b/docs/milestones/milestone-07/FounderOS_Knowledge_Repository_Snapshot_Specification_v1.0.md new file mode 100644 index 0000000..642f0a1 --- /dev/null +++ b/docs/milestones/milestone-07/FounderOS_Knowledge_Repository_Snapshot_Specification_v1.0.md @@ -0,0 +1,26 @@ +# FounderOS Knowledge Repository Snapshot Specification v1.0 + +## Purpose + +Define versioned KnowledgeOS repository snapshots. + +## Snapshot Contains + +- Snapshot ID +- Corpus version +- Source manifest +- Content fingerprint +- Object count +- Creation metadata + +## Requirements + +Snapshots must be: + +- Immutable +- Traceable +- Deterministic + +## Principle + +Versioned knowledge enables reliable future AI context. diff --git a/docs/milestones/milestone-07/FounderOS_Milestone_07_Acceptance_Criteria_v1.0.md b/docs/milestones/milestone-07/FounderOS_Milestone_07_Acceptance_Criteria_v1.0.md new file mode 100644 index 0000000..94d0991 --- /dev/null +++ b/docs/milestones/milestone-07/FounderOS_Milestone_07_Acceptance_Criteria_v1.0.md @@ -0,0 +1,29 @@ +# FounderOS Milestone 07 Acceptance Criteria v1.0 + +## Functional Criteria + +- [ ] Corpus candidate source implemented. +- [ ] Repository initializes from approved corpus. +- [ ] Snapshot identity generated. +- [ ] Provenance preserved. +- [ ] Query behavior remains deterministic. + +## Quality Criteria + +- [ ] Previous milestone tests remain passing. +- [ ] Adapter tests added. +- [ ] Package boundaries preserved. + +## Non Goals + +Not included: + +- Database +- Embeddings +- Semantic retrieval +- Agents +- MCP + +## Definition of Done + +KnowledgeOS operates on a controlled corpus-backed repository snapshot. diff --git a/docs/milestones/milestone-07/FounderOS_Milestone_07_Codex_Execution_Prompt_v1.0.md b/docs/milestones/milestone-07/FounderOS_Milestone_07_Codex_Execution_Prompt_v1.0.md new file mode 100644 index 0000000..69cc453 --- /dev/null +++ b/docs/milestones/milestone-07/FounderOS_Milestone_07_Codex_Execution_Prompt_v1.0.md @@ -0,0 +1,87 @@ +You are the lead engineer responsible for implementing FounderOS +Milestone 07 --- Knowledge Corpus Repository Adapter. + +Before making changes, read: + +- README.md +- AGENTS.md +- CONTRIBUTING.md +- ARCHITECTURE_DECISIONS.md +- Repository audit +- Milestone 04 documents +- Milestone 05 documents +- Milestone 06 documents +- Milestone 07 documents + +Review: + +- packages/knowledge-schema/ +- services/knowledge-engine/ + +Understand: + +- Knowledge Object contracts +- Query contracts +- Repository contracts +- Candidate source boundaries + +Objective: + +Implement a corpus-backed KnowledgeOS repository adapter. + +Move from: + +In-memory repository + +to: + +Approved knowledge corpus source + +without introducing persistence or retrieval intelligence. + +Implement: + +1. Knowledge Corpus Candidate Source +2. Repository snapshot model +3. Snapshot identity +4. Corpus loading workflow +5. Change detection foundation +6. Tests + +Do not implement: + +- Database +- Embeddings +- Vector search +- Semantic ranking +- Knowledge graph +- Agents +- Hermes +- MCP +- UI + +Follow: + +- Documentation first +- Architecture before code +- Preserve package boundaries +- Add tests +- Avoid unnecessary dependencies + +Verification: + +pnpm format:check pnpm lint pnpm build pnpm typecheck pnpm test + +Completion report: + +1. Status GO or NOT READY +2. Summary +3. Changed files +4. Tests +5. Verification results +6. Architecture impact +7. Limitations +8. Next milestone recommendation + +Build a trusted corpus access layer before adding context assembly and +agents. diff --git a/docs/milestones/milestone-07/FounderOS_Milestone_07_Knowledge_Corpus_Repository_Adapter_Specification_v1.0.md b/docs/milestones/milestone-07/FounderOS_Milestone_07_Knowledge_Corpus_Repository_Adapter_Specification_v1.0.md new file mode 100644 index 0000000..fd3729a --- /dev/null +++ b/docs/milestones/milestone-07/FounderOS_Milestone_07_Knowledge_Corpus_Repository_Adapter_Specification_v1.0.md @@ -0,0 +1,53 @@ +# FounderOS Milestone 07 Knowledge Corpus Repository Adapter Specification v1.0 + +## Purpose + +Define the first KnowledgeOS repository adapter connecting the approved FounderOS knowledge corpus to the repository abstraction created in Milestone 06. + +## Objective + +Move from: + + In-memory Repository + | + v + Query Engine + +to: + + Canonical Knowledge Corpus + | + v + Corpus Candidate Source Adapter + | + v + Knowledge Repository + | + v + Query Engine + +## Scope + +Included: + +- Knowledge corpus loading +- Corpus candidate source adapter +- Versioned repository snapshot +- Snapshot identity +- Refresh detection foundation + +Excluded: + +- Database persistence +- Embeddings +- Vector search +- Semantic retrieval +- Ranking +- Knowledge graph +- Agents +- MCP +- UI + +## Principle + +The knowledge corpus should become a controlled source of truth before intelligence layers are added. diff --git a/docs/milestones/milestone-07/FounderOS_Milestone_07_Verification_Checklist_v1.0.md b/docs/milestones/milestone-07/FounderOS_Milestone_07_Verification_Checklist_v1.0.md new file mode 100644 index 0000000..b7c999c --- /dev/null +++ b/docs/milestones/milestone-07/FounderOS_Milestone_07_Verification_Checklist_v1.0.md @@ -0,0 +1,31 @@ +# FounderOS Milestone 07 Verification Checklist v1.0 + +## Architecture Verification + +- [ ] Corpus source uses repository boundary. +- [ ] Query engine remains storage independent. +- [ ] Snapshot model is immutable. + +## Functional Verification + +- [ ] Corpus loads successfully. +- [ ] Provenance is preserved. +- [ ] Results remain deterministic. + +## Change Detection Verification + +- [ ] Snapshot identity is stable. +- [ ] Source changes are detectable. +- [ ] Invalid corpus states fail safely. + +## Engineering Verification + +Run: + +```bash +pnpm format:check +pnpm lint +pnpm build +pnpm typecheck +pnpm test +``` diff --git a/packages/knowledge-schema/README.md b/packages/knowledge-schema/README.md index 108bca5..7613b6c 100644 --- a/packages/knowledge-schema/README.md +++ b/packages/knowledge-schema/README.md @@ -26,6 +26,8 @@ Milestone 05 adds strict, versioned query and result contracts. Queries carry id Milestone 06 adds candidate-source and repository access contracts. A candidate batch binds a validated source descriptor and its provenance to schema-valid Knowledge Objects. The repository interface supports deterministic candidate listing, identity lookup, multi-identity finding, and source inspection. Provider execution and storage behavior remain outside this package. +Milestone 07 adds strict corpus-source, repository-snapshot, and corpus-change contracts. Snapshots bind a corpus version and manifest reference to deterministic object, metadata, source-hash, and content fingerprints. Change sets report version, identity, source, metadata, and object changes without defining refresh execution or persistence behavior. + ## Usage ```typescript diff --git a/packages/knowledge-schema/src/corpus.ts b/packages/knowledge-schema/src/corpus.ts new file mode 100644 index 0000000..5bd9092 --- /dev/null +++ b/packages/knowledge-schema/src/corpus.ts @@ -0,0 +1,247 @@ +import { z } from "zod"; + +import { KnowledgeObjectTypeSchema } from "./enums.js"; +import { MigrationPathSchema, MigrationSourcePathSchema } from "./migration.js"; +import { + IdentifierSchema, + IsoTemporalSchema, + NonEmptyStringSchema, + Sha256DigestSchema, +} from "./primitives.js"; +import { KnowledgeCandidateSourceDescriptorSchema } from "./repository.js"; + +function isSortedUnique(values: readonly string[]): boolean { + return ( + new Set(values).size === values.length && + values.every((value, index) => index === 0 || values[index - 1]! < value) + ); +} + +function requireSortedUniqueBy( + values: readonly T[], + key: (value: T) => string, + context: z.RefinementCtx, + path: string, +): void { + if (!isSortedUnique(values.map(key))) { + context.addIssue({ + code: "custom", + message: `${path} must be unique and sorted`, + path: [path], + }); + } +} + +export const KnowledgeCorpusSourceSchema = z + .object({ + schemaVersion: z.literal("1.0"), + corpusId: IdentifierSchema, + corpusVersion: NonEmptyStringSchema, + sourceManifestReference: MigrationPathSchema, + source: KnowledgeCandidateSourceDescriptorSchema, + }) + .strict() + .superRefine((value, context) => { + if (value.source.sourceType !== "knowledge_corpus") { + context.addIssue({ + code: "custom", + message: "Corpus candidate sources must use sourceType knowledge_corpus", + path: ["source", "sourceType"], + }); + } + if (value.source.provenance.sourceReference !== value.sourceManifestReference) { + context.addIssue({ + code: "custom", + message: "Candidate source provenance must reference the source manifest", + path: ["source", "provenance", "sourceReference"], + }); + } + }); + +export const KnowledgeRepositorySnapshotObjectSchema = z + .object({ + objectId: IdentifierSchema, + objectType: KnowledgeObjectTypeSchema, + sourcePath: MigrationSourcePathSchema, + sourceHash: Sha256DigestSchema, + metadataFingerprint: Sha256DigestSchema, + objectFingerprint: Sha256DigestSchema, + }) + .strict(); + +export const KnowledgeRepositorySnapshotCreationSchema = z + .object({ + createdAt: IsoTemporalSchema, + createdBy: IdentifierSchema, + }) + .strict(); + +export const KnowledgeRepositorySnapshotSchema = z + .object({ + schemaVersion: z.literal("1.0"), + snapshotId: IdentifierSchema, + corpusId: IdentifierSchema, + corpusVersion: NonEmptyStringSchema, + sourceManifestReference: MigrationPathSchema, + contentFingerprint: Sha256DigestSchema, + objectCount: z.number().int().nonnegative(), + creation: KnowledgeRepositorySnapshotCreationSchema, + objects: z.array(KnowledgeRepositorySnapshotObjectSchema), + }) + .strict() + .superRefine((snapshot, context) => { + if (snapshot.objectCount !== snapshot.objects.length) { + context.addIssue({ + code: "custom", + message: "objectCount must equal the number of snapshot object records", + path: ["objectCount"], + }); + } + if (snapshot.snapshotId !== `snapshot-${snapshot.contentFingerprint}`) { + context.addIssue({ + code: "custom", + message: "snapshotId must be derived from contentFingerprint", + path: ["snapshotId"], + }); + } + requireSortedUniqueBy(snapshot.objects, (object) => object.objectId, context, "objects"); + const paths = snapshot.objects.map((object) => object.sourcePath); + if (new Set(paths).size !== paths.length) { + context.addIssue({ + code: "custom", + message: "Snapshot source paths must be unique", + path: ["objects"], + }); + } + }); + +export const KnowledgeCorpusIdentityChangeSchema = z + .object({ + sourcePath: MigrationSourcePathSchema, + previousObjectId: IdentifierSchema, + currentObjectId: IdentifierSchema, + }) + .strict() + .refine((change) => change.previousObjectId !== change.currentObjectId, { + message: "Identity changes must contain different object IDs", + path: ["currentObjectId"], + }); + +function fingerprintChangeSchema(fieldName: string) { + return z + .object({ + objectId: IdentifierSchema, + previous: Sha256DigestSchema, + current: Sha256DigestSchema, + }) + .strict() + .refine((change) => change.previous !== change.current, { + message: `${fieldName} changes must contain different fingerprints`, + path: ["current"], + }); +} + +export const KnowledgeCorpusSourceHashChangeSchema = fingerprintChangeSchema("sourceHash"); +export const KnowledgeCorpusMetadataChangeSchema = fingerprintChangeSchema("metadataFingerprint"); +export const KnowledgeCorpusObjectChangeSchema = fingerprintChangeSchema("objectFingerprint"); + +export const KnowledgeCorpusChangeSetSchema = z + .object({ + schemaVersion: z.literal("1.0"), + previousSnapshotId: IdentifierSchema, + currentSnapshotId: IdentifierSchema, + previousCorpusVersion: NonEmptyStringSchema, + currentCorpusVersion: NonEmptyStringSchema, + corpusVersionChanged: z.boolean(), + previousContentFingerprint: Sha256DigestSchema, + currentContentFingerprint: Sha256DigestSchema, + contentFingerprintChanged: z.boolean(), + addedObjectIds: z.array(IdentifierSchema), + removedObjectIds: z.array(IdentifierSchema), + identityChanges: z.array(KnowledgeCorpusIdentityChangeSchema), + sourceHashChanges: z.array(KnowledgeCorpusSourceHashChangeSchema), + metadataChanges: z.array(KnowledgeCorpusMetadataChangeSchema), + objectChanges: z.array(KnowledgeCorpusObjectChangeSchema), + changed: z.boolean(), + }) + .strict() + .superRefine((value, context) => { + if ( + value.corpusVersionChanged !== + (value.previousCorpusVersion !== value.currentCorpusVersion) + ) { + context.addIssue({ + code: "custom", + message: "corpusVersionChanged must match the compared corpus versions", + path: ["corpusVersionChanged"], + }); + } + if ( + value.contentFingerprintChanged !== + (value.previousContentFingerprint !== value.currentContentFingerprint) + ) { + context.addIssue({ + code: "custom", + message: "contentFingerprintChanged must match the compared content fingerprints", + path: ["contentFingerprintChanged"], + }); + } + if (!isSortedUnique(value.addedObjectIds)) + context.addIssue({ + code: "custom", + message: "addedObjectIds must be unique and sorted", + path: ["addedObjectIds"], + }); + if (!isSortedUnique(value.removedObjectIds)) + context.addIssue({ + code: "custom", + message: "removedObjectIds must be unique and sorted", + path: ["removedObjectIds"], + }); + requireSortedUniqueBy( + value.identityChanges, + (change) => change.sourcePath, + context, + "identityChanges", + ); + requireSortedUniqueBy( + value.sourceHashChanges, + (change) => change.objectId, + context, + "sourceHashChanges", + ); + requireSortedUniqueBy( + value.metadataChanges, + (change) => change.objectId, + context, + "metadataChanges", + ); + requireSortedUniqueBy( + value.objectChanges, + (change) => change.objectId, + context, + "objectChanges", + ); + const detected = + value.corpusVersionChanged || + value.contentFingerprintChanged || + value.addedObjectIds.length > 0 || + value.removedObjectIds.length > 0 || + value.identityChanges.length > 0 || + value.sourceHashChanges.length > 0 || + value.metadataChanges.length > 0 || + value.objectChanges.length > 0; + if (value.changed !== detected) + context.addIssue({ + code: "custom", + message: "changed must reflect whether the comparison contains any change", + path: ["changed"], + }); + }); + +export type KnowledgeCorpusSource = z.infer; +export type KnowledgeRepositorySnapshotObject = z.infer< + typeof KnowledgeRepositorySnapshotObjectSchema +>; +export type KnowledgeRepositorySnapshot = z.infer; +export type KnowledgeCorpusChangeSet = z.infer; diff --git a/packages/knowledge-schema/src/index.ts b/packages/knowledge-schema/src/index.ts index 790fcf1..e2b54d4 100644 --- a/packages/knowledge-schema/src/index.ts +++ b/packages/knowledge-schema/src/index.ts @@ -1,3 +1,4 @@ +export * from "./corpus.js"; export * from "./enums.js"; export * from "./metadata.js"; export * from "./migration.js"; diff --git a/packages/knowledge-schema/src/migration.ts b/packages/knowledge-schema/src/migration.ts index 065b4f1..4618c43 100644 --- a/packages/knowledge-schema/src/migration.ts +++ b/packages/knowledge-schema/src/migration.ts @@ -8,9 +8,12 @@ import { KnowledgeStatusSchema, } from "./enums.js"; import { RelationshipReferenceSchema } from "./metadata.js"; -import { IdentifierSchema, IsoTemporalSchema, NonEmptyStringSchema } from "./primitives.js"; - -const SHA256_PATTERN = /^[a-f0-9]{64}$/u; +import { + IdentifierSchema, + IsoTemporalSchema, + NonEmptyStringSchema, + Sha256DigestSchema, +} from "./primitives.js"; function isSafeRelativePath(value: string): boolean { if (value.startsWith("/") || value.includes("\\") || value.includes("\0")) { @@ -66,7 +69,7 @@ export const KnowledgeMigrationManifestEntrySchema = z objectType: KnowledgeObjectTypeSchema, sourcePath: MigrationSourcePathSchema, destinationPath: MigrationDestinationPathSchema, - sourceHash: z.string().regex(SHA256_PATTERN, "Expected a lowercase SHA-256 digest"), + sourceHash: Sha256DigestSchema, migrationStatus: MigrationStatusSchema, reviewStatus: ReviewStatusSchema, metadata: MigrationMetadataSchema, diff --git a/packages/knowledge-schema/src/parse.ts b/packages/knowledge-schema/src/parse.ts index 9a5d69c..c8f2d6b 100644 --- a/packages/knowledge-schema/src/parse.ts +++ b/packages/knowledge-schema/src/parse.ts @@ -1,3 +1,11 @@ +import { + KnowledgeCorpusChangeSetSchema, + KnowledgeCorpusSourceSchema, + KnowledgeRepositorySnapshotSchema, + type KnowledgeCorpusChangeSet, + type KnowledgeCorpusSource, + type KnowledgeRepositorySnapshot, +} from "./corpus.js"; import { KnowledgeMetadataSchema, type KnowledgeMetadata } from "./metadata.js"; import { KnowledgeObjectSchema, type KnowledgeObject } from "./objects.js"; import { KnowledgeQuerySchema, type KnowledgeQuery } from "./query.js"; @@ -70,3 +78,27 @@ export function parseKnowledgeRepositoryFindRequest( export function safeParseKnowledgeRepositoryFindRequest(input: unknown) { return KnowledgeRepositoryFindRequestSchema.safeParse(input); } + +export function parseKnowledgeCorpusSource(input: unknown): KnowledgeCorpusSource { + return KnowledgeCorpusSourceSchema.parse(input); +} + +export function safeParseKnowledgeCorpusSource(input: unknown) { + return KnowledgeCorpusSourceSchema.safeParse(input); +} + +export function parseKnowledgeRepositorySnapshot(input: unknown): KnowledgeRepositorySnapshot { + return KnowledgeRepositorySnapshotSchema.parse(input); +} + +export function safeParseKnowledgeRepositorySnapshot(input: unknown) { + return KnowledgeRepositorySnapshotSchema.safeParse(input); +} + +export function parseKnowledgeCorpusChangeSet(input: unknown): KnowledgeCorpusChangeSet { + return KnowledgeCorpusChangeSetSchema.parse(input); +} + +export function safeParseKnowledgeCorpusChangeSet(input: unknown) { + return KnowledgeCorpusChangeSetSchema.safeParse(input); +} diff --git a/packages/knowledge-schema/src/primitives.ts b/packages/knowledge-schema/src/primitives.ts index 0f0c196..e056f46 100644 --- a/packages/knowledge-schema/src/primitives.ts +++ b/packages/knowledge-schema/src/primitives.ts @@ -2,6 +2,9 @@ import { z } from "zod"; export const NonEmptyStringSchema = z.string().trim().min(1); export const IdentifierSchema = NonEmptyStringSchema; +export const Sha256DigestSchema = z + .string() + .regex(/^[a-f0-9]{64}$/u, "Expected a lowercase SHA-256 digest"); const ISO_DATE_PATTERN = /^\d{4}-\d{2}-\d{2}$/u; const ISO_DATE_TIME_PATTERN = diff --git a/packages/knowledge-schema/tests/corpus.test.ts b/packages/knowledge-schema/tests/corpus.test.ts new file mode 100644 index 0000000..3a99c5e --- /dev/null +++ b/packages/knowledge-schema/tests/corpus.test.ts @@ -0,0 +1,200 @@ +import { describe, expect, it } from "vitest"; + +import { + KnowledgeCorpusChangeSetSchema, + KnowledgeCorpusSourceSchema, + KnowledgeRepositorySnapshotSchema, + safeParseKnowledgeCorpusChangeSet, + safeParseKnowledgeRepositorySnapshot, +} from "../src/index.js"; + +const A = "a".repeat(64); +const B = "b".repeat(64); + +function source() { + return { + schemaVersion: "1.0", + corpusId: "founderos-priority-1", + corpusVersion: "1.0", + sourceManifestReference: "knowledge/migration-manifest.yaml", + source: { + schemaVersion: "1.0", + sourceId: "founderos-priority-1", + sourceType: "knowledge_corpus", + provenance: { + sourceType: "migration_manifest", + sourceReference: "knowledge/migration-manifest.yaml", + originalCreator: "FounderOS", + }, + }, + }; +} + +function snapshot() { + return { + schemaVersion: "1.0", + snapshotId: `snapshot-${A}`, + corpusId: "founderos-priority-1", + corpusVersion: "1.0", + sourceManifestReference: "knowledge/migration-manifest.yaml", + contentFingerprint: A, + objectCount: 2, + creation: { createdAt: "2026-07-28T00:00:00Z", createdBy: "knowledge-engine" }, + objects: [ + { + objectId: "a", + objectType: "knowledge", + sourcePath: "docs/a.md", + sourceHash: A, + metadataFingerprint: A, + objectFingerprint: A, + }, + { + objectId: "b", + objectType: "principle", + sourcePath: "docs/b.md", + sourceHash: B, + metadataFingerprint: B, + objectFingerprint: B, + }, + ], + }; +} + +function unchanged() { + return { + schemaVersion: "1.0", + previousSnapshotId: `snapshot-${A}`, + currentSnapshotId: `snapshot-${A}`, + previousCorpusVersion: "1.0", + currentCorpusVersion: "1.0", + corpusVersionChanged: false, + previousContentFingerprint: A, + currentContentFingerprint: A, + contentFingerprintChanged: false, + addedObjectIds: [], + removedObjectIds: [], + identityChanges: [], + sourceHashChanges: [], + metadataChanges: [], + objectChanges: [], + changed: false, + }; +} + +describe("Milestone 07 corpus contracts", () => { + it("validates corpus source identity, version, and manifest provenance", () => { + expect(KnowledgeCorpusSourceSchema.parse(source())).toEqual(source()); + expect( + KnowledgeCorpusSourceSchema.safeParse({ ...source(), storageUrl: "private" }).success, + ).toBe(false); + expect( + KnowledgeCorpusSourceSchema.safeParse({ + ...source(), + source: { ...source().source, sourceType: "in_memory" }, + }).success, + ).toBe(false); + }); + + it("validates deterministic snapshot records", () => { + expect(KnowledgeRepositorySnapshotSchema.parse(snapshot())).toEqual(snapshot()); + expect(safeParseKnowledgeRepositorySnapshot({ ...snapshot(), objectCount: 1 }).success).toBe( + false, + ); + expect( + safeParseKnowledgeRepositorySnapshot({ ...snapshot(), contentFingerprint: "bad" }).success, + ).toBe(false); + expect( + safeParseKnowledgeRepositorySnapshot({ ...snapshot(), snapshotId: "snapshot-arbitrary" }) + .success, + ).toBe(false); + }); + + it("rejects duplicate IDs, duplicate paths, and unsorted snapshot records", () => { + const duplicateId = snapshot(); + duplicateId.objects[1]!.objectId = "a"; + expect(safeParseKnowledgeRepositorySnapshot(duplicateId).success).toBe(false); + const duplicatePath = snapshot(); + duplicatePath.objects[1]!.sourcePath = "docs/a.md"; + expect(safeParseKnowledgeRepositorySnapshot(duplicatePath).success).toBe(false); + const unsorted = snapshot(); + unsorted.objects.reverse(); + expect(safeParseKnowledgeRepositorySnapshot(unsorted).success).toBe(false); + }); + + it("validates unchanged and changed result consistency", () => { + expect(KnowledgeCorpusChangeSetSchema.parse(unchanged()).changed).toBe(false); + expect(safeParseKnowledgeCorpusChangeSet({ ...unchanged(), changed: true }).success).toBe( + false, + ); + expect( + safeParseKnowledgeCorpusChangeSet({ ...unchanged(), corpusVersionChanged: true }).success, + ).toBe(false); + expect( + safeParseKnowledgeCorpusChangeSet({ ...unchanged(), contentFingerprintChanged: true }) + .success, + ).toBe(false); + expect(safeParseKnowledgeCorpusChangeSet({ ...unchanged(), databaseRevision: 1 }).success).toBe( + false, + ); + }); + + it.each([ + ["addedObjectIds", { addedObjectIds: ["b", "a"], changed: true }], + ["removedObjectIds", { removedObjectIds: ["a", "a"], changed: true }], + [ + "identityChanges", + { + identityChanges: [ + { sourcePath: "docs/a.md", previousObjectId: "a", currentObjectId: "b" }, + { sourcePath: "docs/a.md", previousObjectId: "c", currentObjectId: "d" }, + ], + changed: true, + }, + ], + [ + "sourceHashChanges", + { + sourceHashChanges: [ + { objectId: "a", previous: A, current: B }, + { objectId: "a", previous: B, current: A }, + ], + changed: true, + }, + ], + [ + "metadataChanges", + { + metadataChanges: [ + { objectId: "z", previous: A, current: B }, + { objectId: "a", previous: A, current: B }, + ], + changed: true, + }, + ], + [ + "objectChanges", + { + objectChanges: [ + { objectId: "a", previous: A, current: B }, + { objectId: "a", previous: B, current: A }, + ], + changed: true, + }, + ], + ])("rejects invalid deterministic ordering in %s", (_label, replacement) => { + expect(safeParseKnowledgeCorpusChangeSet({ ...unchanged(), ...replacement }).success).toBe( + false, + ); + }); + + it("rejects unchanged fingerprint entries", () => { + expect( + safeParseKnowledgeCorpusChangeSet({ + ...unchanged(), + sourceHashChanges: [{ objectId: "a", previous: A, current: A }], + changed: true, + }).success, + ).toBe(false); + }); +}); diff --git a/services/knowledge-engine/README.md b/services/knowledge-engine/README.md index 74cf299..093201f 100644 --- a/services/knowledge-engine/README.md +++ b/services/knowledge-engine/README.md @@ -6,7 +6,9 @@ Milestone 04 adds manifest-controlled corpus execution. It loads a strict YAML m Milestone 05 adds a pure, storage-free query boundary over an explicitly supplied set of validated Knowledge Objects. It validates the query and every candidate, rejects duplicate object IDs, applies exact filters and context constraints by intersection, and sorts results by object ID. Multi-value filters match any allowed value; tag filters explicitly support `all` or `any`. Project filters match an object's domain, a project object's ID or name, or a decision object's `relatedProjectIds`. Every result includes the object's unchanged source metadata as provenance. -Milestone 06 makes repository-backed querying the primary access flow. Candidate sources emit versioned batches; the in-memory repository revalidates them, rejects duplicate source or object identities, builds a deterministic snapshot, and returns independent validated copies. It provides identity lookup and candidate discovery without persistence or search intelligence. The Milestone 05 candidate-array function remains available as the compatibility filtering core. +Milestone 06 makes repository-backed querying the primary access flow. Candidate sources emit versioned batches; the in-memory repository revalidates them, rejects duplicate source or object identities, builds a deterministic in-memory collection, and returns independent validated copies. It provides identity lookup and candidate discovery without persistence or search intelligence. The Milestone 05 candidate-array function remains available as the compatibility filtering core. + +Milestone 07 connects the approved Priority 1 corpus to that repository boundary. `KnowledgeCorpusCandidateSource` reuses the manifest-controlled migration workflow, rejects partially valid corpus states, preserves object provenance, and creates an immutable repository snapshot with a content-derived identity. `initializeCorpusKnowledgeRepository` exposes the unchanged repository query capability, while `compareKnowledgeRepositorySnapshots` reports deterministic corpus changes without performing refreshes. ```typescript import { queryKnowledgeObjects } from "@founderos/knowledge-engine"; @@ -26,10 +28,22 @@ const repository = await InMemoryKnowledgeRepository.create([source]); const result = await queryKnowledgeRepository(query, repository); ``` +```typescript +import { initializeCorpusKnowledgeRepository } from "@founderos/knowledge-engine"; + +const { repository, snapshot } = await initializeCorpusKnowledgeRepository({ + rootPath: process.cwd(), + manifestPath: "knowledge/migration-manifest.yaml", + corpusVersion: "priority-1-v1", + createdAt: "2026-07-28T00:00:00Z", + createdBy: "knowledge-engine", +}); +``` + From the repository root: ```bash pnpm knowledge:migrate ``` -Directory ingestion is recursive, Markdown-only, stable in path order, and does not follow symbolic links. Repository access is an immutable in-memory snapshot, not durable storage. Querying remains deterministic exact filtering—not full-text search, semantic retrieval, ranking, or authorization. The implementation remains read-only and does not watch a vault or implement persistence, embeddings, graph storage, Hermes, agents, or MCP integrations. +Directory ingestion is recursive, Markdown-only, stable in path order, and does not follow symbolic links. Repository access is an immutable in-memory snapshot, not durable storage. Snapshot comparison detects changes but does not synchronize them. Querying remains deterministic exact filtering—not full-text search, semantic retrieval, ranking, or authorization. The implementation remains read-only and does not watch a vault or implement persistence, embeddings, graph storage, Hermes, agents, or MCP integrations. diff --git a/services/knowledge-engine/src/application/compare-knowledge-repository-snapshots.ts b/services/knowledge-engine/src/application/compare-knowledge-repository-snapshots.ts new file mode 100644 index 0000000..e05a559 --- /dev/null +++ b/services/knowledge-engine/src/application/compare-knowledge-repository-snapshots.ts @@ -0,0 +1,132 @@ +import { + KnowledgeCorpusChangeSetSchema, + KnowledgeRepositorySnapshotSchema, + type KnowledgeCorpusChangeSet, + type KnowledgeRepositorySnapshot, + type KnowledgeRepositorySnapshotObject, +} from "@founderos/knowledge-schema"; + +export class KnowledgeCorpusComparisonError extends Error { + public constructor(previousCorpusId: string, currentCorpusId: string) { + super( + `Cannot compare repository snapshots from different corpora: ${previousCorpusId} and ${currentCorpusId}`, + ); + this.name = "KnowledgeCorpusComparisonError"; + } +} + +function compareStrings(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +function indexById( + objects: readonly KnowledgeRepositorySnapshotObject[], +): Map { + return new Map(objects.map((object) => [object.objectId, object])); +} + +export function compareKnowledgeRepositorySnapshots( + previousInput: KnowledgeRepositorySnapshot, + currentInput: KnowledgeRepositorySnapshot, +): KnowledgeCorpusChangeSet { + const previous = KnowledgeRepositorySnapshotSchema.parse(previousInput); + const current = KnowledgeRepositorySnapshotSchema.parse(currentInput); + + if (previous.corpusId !== current.corpusId) { + throw new KnowledgeCorpusComparisonError(previous.corpusId, current.corpusId); + } + + const previousById = indexById(previous.objects); + const currentById = indexById(current.objects); + const previousByPath = new Map(previous.objects.map((object) => [object.sourcePath, object])); + const currentByPath = new Map(current.objects.map((object) => [object.sourcePath, object])); + const addedObjectIds = current.objects + .filter((object) => !previousById.has(object.objectId)) + .map((object) => object.objectId) + .sort(compareStrings); + const removedObjectIds = previous.objects + .filter((object) => !currentById.has(object.objectId)) + .map((object) => object.objectId) + .sort(compareStrings); + const identityChanges = [...previousByPath.entries()] + .flatMap(([sourcePath, previousObject]) => { + const currentObject = currentByPath.get(sourcePath); + return currentObject !== undefined && currentObject.objectId !== previousObject.objectId + ? [ + { + sourcePath, + previousObjectId: previousObject.objectId, + currentObjectId: currentObject.objectId, + }, + ] + : []; + }) + .sort((left, right) => compareStrings(left.sourcePath, right.sourcePath)); + const sharedObjectIds = previous.objects + .map((object) => object.objectId) + .filter((objectId) => currentById.has(objectId)) + .sort(compareStrings); + const sourceHashChanges = sharedObjectIds.flatMap((objectId) => { + const previousObject = previousById.get(objectId)!; + const currentObject = currentById.get(objectId)!; + return previousObject.sourceHash !== currentObject.sourceHash + ? [{ objectId, previous: previousObject.sourceHash, current: currentObject.sourceHash }] + : []; + }); + const metadataChanges = sharedObjectIds.flatMap((objectId) => { + const previousObject = previousById.get(objectId)!; + const currentObject = currentById.get(objectId)!; + return previousObject.metadataFingerprint !== currentObject.metadataFingerprint + ? [ + { + objectId, + previous: previousObject.metadataFingerprint, + current: currentObject.metadataFingerprint, + }, + ] + : []; + }); + const objectChanges = sharedObjectIds.flatMap((objectId) => { + const previousObject = previousById.get(objectId)!; + const currentObject = currentById.get(objectId)!; + return previousObject.objectFingerprint !== currentObject.objectFingerprint + ? [ + { + objectId, + previous: previousObject.objectFingerprint, + current: currentObject.objectFingerprint, + }, + ] + : []; + }); + const corpusVersionChanged = previous.corpusVersion !== current.corpusVersion; + const contentFingerprintChanged = previous.contentFingerprint !== current.contentFingerprint; + const changed = + corpusVersionChanged || + contentFingerprintChanged || + addedObjectIds.length > 0 || + removedObjectIds.length > 0 || + identityChanges.length > 0 || + sourceHashChanges.length > 0 || + metadataChanges.length > 0 || + objectChanges.length > 0; + + return KnowledgeCorpusChangeSetSchema.parse({ + schemaVersion: "1.0", + previousSnapshotId: previous.snapshotId, + currentSnapshotId: current.snapshotId, + previousCorpusVersion: previous.corpusVersion, + currentCorpusVersion: current.corpusVersion, + corpusVersionChanged, + previousContentFingerprint: previous.contentFingerprint, + currentContentFingerprint: current.contentFingerprint, + contentFingerprintChanged, + addedObjectIds, + removedObjectIds, + identityChanges, + sourceHashChanges, + metadataChanges, + objectChanges, + changed, + }); +} diff --git a/services/knowledge-engine/src/application/initialize-corpus-knowledge-repository.ts b/services/knowledge-engine/src/application/initialize-corpus-knowledge-repository.ts new file mode 100644 index 0000000..6692bfa --- /dev/null +++ b/services/knowledge-engine/src/application/initialize-corpus-knowledge-repository.ts @@ -0,0 +1,22 @@ +import type { KnowledgeRepositorySnapshot } from "@founderos/knowledge-schema"; + +import { + KnowledgeCorpusCandidateSource, + type CreateKnowledgeCorpusCandidateSourceOptions, +} from "../infrastructure/knowledge-corpus-candidate-source.js"; +import { InMemoryKnowledgeRepository } from "../infrastructure/in-memory-knowledge-repository.js"; + +export interface InitializedCorpusKnowledgeRepository { + candidateSource: KnowledgeCorpusCandidateSource; + repository: InMemoryKnowledgeRepository; + snapshot: KnowledgeRepositorySnapshot; +} + +export async function initializeCorpusKnowledgeRepository( + options: CreateKnowledgeCorpusCandidateSourceOptions, +): Promise { + const candidateSource = await KnowledgeCorpusCandidateSource.create(options); + const repository = await InMemoryKnowledgeRepository.create([candidateSource]); + + return { candidateSource, repository, snapshot: candidateSource.snapshot }; +} diff --git a/services/knowledge-engine/src/index.ts b/services/knowledge-engine/src/index.ts index 79ae73f..2a6ae4b 100644 --- a/services/knowledge-engine/src/index.ts +++ b/services/knowledge-engine/src/index.ts @@ -1,6 +1,8 @@ export * from "./application/ingest-markdown.js"; export * from "./application/ingest-markdown-directory.js"; export * from "./application/execute-knowledge-migration.js"; +export * from "./application/compare-knowledge-repository-snapshots.js"; +export * from "./application/initialize-corpus-knowledge-repository.js"; export * from "./application/normalize-frontmatter.js"; export * from "./application/query-knowledge.js"; export * from "./application/query-knowledge-repository.js"; @@ -9,6 +11,7 @@ export * from "./domain/frontmatter.js"; export * from "./domain/knowledge-query.js"; export * from "./domain/safe-path.js"; export * from "./infrastructure/load-migration-manifest.js"; +export * from "./infrastructure/knowledge-corpus-candidate-source.js"; export * from "./infrastructure/in-memory-candidate-source.js"; export * from "./infrastructure/in-memory-knowledge-repository.js"; export * from "./infrastructure/parse-markdown.js"; diff --git a/services/knowledge-engine/src/infrastructure/knowledge-corpus-candidate-source.ts b/services/knowledge-engine/src/infrastructure/knowledge-corpus-candidate-source.ts new file mode 100644 index 0000000..31f0f34 --- /dev/null +++ b/services/knowledge-engine/src/infrastructure/knowledge-corpus-candidate-source.ts @@ -0,0 +1,198 @@ +import { createHash } from "node:crypto"; + +import { + KnowledgeCandidateBatchSchema, + KnowledgeCorpusSourceSchema, + KnowledgeRepositorySnapshotCreationSchema, + KnowledgeRepositorySnapshotSchema, + MigrationPathSchema, + NonEmptyStringSchema, + type KnowledgeCandidateBatch, + type KnowledgeCandidateSource, + type KnowledgeCorpusSource, + type KnowledgeRepositorySnapshot, + type KnowledgeRepositorySnapshotObject, +} from "@founderos/knowledge-schema"; + +import { executeKnowledgeMigration } from "../application/execute-knowledge-migration.js"; +import type { + AcceptedMigrationDocumentReport, + KnowledgeMigrationReport, +} from "../interfaces/migration-report.js"; + +function compareStrings(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +function canonicalize(value: unknown): string { + if (value === null || typeof value !== "object") { + return JSON.stringify(value) ?? "null"; + } + + if (Array.isArray(value)) { + return `[${value.map(canonicalize).join(",")}]`; + } + + const entries = Object.entries(value) + .filter(([, entryValue]) => entryValue !== undefined) + .sort(([left], [right]) => compareStrings(left, right)); + + return `{${entries + .map(([key, entryValue]) => `${JSON.stringify(key)}:${canonicalize(entryValue)}`) + .join(",")}}`; +} + +function sha256(value: unknown): string { + return createHash("sha256").update(canonicalize(value)).digest("hex"); +} + +function deepFreeze(value: T, visited = new WeakSet()): T { + if (value === null || typeof value !== "object" || visited.has(value)) return value; + + visited.add(value); + for (const child of Object.values(value)) deepFreeze(child, visited); + return Object.freeze(value); +} + +export class KnowledgeCorpusMigrationRejectedError extends Error { + public readonly report: KnowledgeMigrationReport; + + public constructor(report: KnowledgeMigrationReport) { + const rejectedDocumentCount = report.documents.filter( + (document) => document.status === "rejected", + ).length; + super( + `Knowledge corpus migration was rejected (${report.errors.length} manifest error(s), ${rejectedDocumentCount} rejected document(s))`, + ); + this.name = "KnowledgeCorpusMigrationRejectedError"; + this.report = report; + } +} + +export interface CreateKnowledgeRepositorySnapshotInput { + corpus: KnowledgeCorpusSource; + creation: KnowledgeRepositorySnapshot["creation"]; + documents: readonly AcceptedMigrationDocumentReport[]; +} + +export function createKnowledgeRepositorySnapshot( + input: CreateKnowledgeRepositorySnapshotInput, +): KnowledgeRepositorySnapshot { + const corpus = KnowledgeCorpusSourceSchema.parse(input.corpus); + const creation = KnowledgeRepositorySnapshotCreationSchema.parse(input.creation); + const objects: KnowledgeRepositorySnapshotObject[] = input.documents + .map((document) => ({ + metadataFingerprint: sha256(document.object.metadata), + objectFingerprint: sha256(document.object), + objectId: document.object.metadata.id, + objectType: document.object.metadata.objectType, + sourceHash: document.actualSourceHash, + sourcePath: document.sourcePath, + })) + .sort((left, right) => compareStrings(left.objectId, right.objectId)); + const contentFingerprint = sha256({ + corpusId: corpus.corpusId, + corpusVersion: corpus.corpusVersion, + objects, + sourceManifestReference: corpus.sourceManifestReference, + }); + const snapshot = KnowledgeRepositorySnapshotSchema.parse({ + schemaVersion: "1.0", + snapshotId: `snapshot-${contentFingerprint}`, + corpusId: corpus.corpusId, + corpusVersion: corpus.corpusVersion, + sourceManifestReference: corpus.sourceManifestReference, + contentFingerprint, + objectCount: objects.length, + creation, + objects, + }); + + return deepFreeze(snapshot); +} + +export interface CreateKnowledgeCorpusCandidateSourceOptions { + rootPath: string; + manifestPath: string; + corpusVersion: string; + createdAt: string; + createdBy: string; +} + +export class KnowledgeCorpusCandidateSource implements KnowledgeCandidateSource { + readonly #batch: KnowledgeCandidateBatch; + public readonly corpus: KnowledgeCorpusSource; + public readonly report: KnowledgeMigrationReport; + public readonly snapshot: KnowledgeRepositorySnapshot; + + private constructor( + batch: KnowledgeCandidateBatch, + corpus: KnowledgeCorpusSource, + report: KnowledgeMigrationReport, + snapshot: KnowledgeRepositorySnapshot, + ) { + this.#batch = batch; + this.corpus = corpus; + this.report = report; + this.snapshot = snapshot; + } + + public static async create( + options: CreateKnowledgeCorpusCandidateSourceOptions, + ): Promise { + const rootPath = NonEmptyStringSchema.parse(options.rootPath); + const manifestPath = MigrationPathSchema.parse(options.manifestPath); + const corpusVersion = NonEmptyStringSchema.parse(options.corpusVersion); + const creation = KnowledgeRepositorySnapshotCreationSchema.parse({ + createdAt: options.createdAt, + createdBy: options.createdBy, + }); + const report = await executeKnowledgeMigration({ rootPath, manifestPath }); + const acceptedDocuments = report.documents.filter( + (document): document is AcceptedMigrationDocumentReport => document.status === "accepted", + ); + + if ( + report.status !== "accepted" || + report.corpusId === null || + acceptedDocuments.length !== report.documents.length + ) { + throw new KnowledgeCorpusMigrationRejectedError(report); + } + + const corpus = KnowledgeCorpusSourceSchema.parse({ + schemaVersion: "1.0", + corpusId: report.corpusId, + corpusVersion, + sourceManifestReference: report.manifestPath, + source: { + schemaVersion: "1.0", + sourceId: report.corpusId, + sourceType: "knowledge_corpus", + provenance: { + sourceType: "migration_manifest", + sourceReference: report.manifestPath, + originalCreator: "FounderOS", + }, + }, + }); + const batch = KnowledgeCandidateBatchSchema.parse({ + schemaVersion: "1.0", + source: corpus.source, + candidates: acceptedDocuments + .map((document) => document.object) + .sort((left, right) => compareStrings(left.metadata.id, right.metadata.id)), + }); + const snapshot = createKnowledgeRepositorySnapshot({ + corpus, + creation, + documents: acceptedDocuments, + }); + + return new KnowledgeCorpusCandidateSource(batch, corpus, report, snapshot); + } + + public async loadCandidates(): Promise { + return KnowledgeCandidateBatchSchema.parse(this.#batch); + } +} diff --git a/services/knowledge-engine/tests/knowledge-corpus-candidate-source.test.ts b/services/knowledge-engine/tests/knowledge-corpus-candidate-source.test.ts new file mode 100644 index 0000000..5a0e7f9 --- /dev/null +++ b/services/knowledge-engine/tests/knowledge-corpus-candidate-source.test.ts @@ -0,0 +1,305 @@ +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import type { KnowledgeRepositorySnapshot } from "@founderos/knowledge-schema"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { stringify } from "yaml"; + +import { + compareKnowledgeRepositorySnapshots, + createKnowledgeRepositorySnapshot, + initializeCorpusKnowledgeRepository, + KnowledgeCorpusCandidateSource, + KnowledgeCorpusMigrationRejectedError, + queryKnowledgeObjects, + queryKnowledgeRepository, + serializeKnowledgeQueryResult, +} from "../src/index.js"; +import type { AcceptedMigrationDocumentReport } from "../src/interfaces/migration-report.js"; +import { PRIORITY_ONE_QUERY_EVALUATIONS } from "./fixtures/query-evaluations.js"; + +const REPOSITORY_ROOT = resolve(fileURLToPath(new URL("../../../", import.meta.url))); +const MANIFEST_PATH = "knowledge/migration-manifest.yaml"; +const CREATION = { createdAt: "2026-07-28T00:00:00Z", createdBy: "founderos-engine" } as const; +const PRIORITY_ONE_SOURCE_PATHS = [ + "docs/architecture/FounderOS_Data_Architecture_Specification_v1.0.md", + "docs/architecture/FounderOS_MCP_Architecture_Specification_v1.0.md", + "docs/architecture/FounderOS_Repository_Architecture_Specification_v1.0.md", + "docs/architecture/FounderOS_Security_and_Governance_Architecture_Specification_v1.0.md", + "docs/architecture/FounderOS_System_Architecture_Specification_v1.0.md", + "docs/governance/FounderOS_Constitution_v1.0.md", + "docs/governance/FounderOS_Decision_Framework_v1.0.md", + "docs/governance/FounderOS_Design_Principles_v1.0.md", +] as const; + +let initialized: Awaited>; +let canonicalBytesBefore: Buffer[]; +const temporaryDirectories: string[] = []; + +function options(creation: { createdAt: string; createdBy: string } = CREATION) { + return { + rootPath: REPOSITORY_ROOT, + manifestPath: MANIFEST_PATH, + corpusVersion: "priority-1-v1", + ...creation, + }; +} + +function hash(source: string): string { + return createHash("sha256").update(source).digest("hex"); +} + +function acceptedDocuments(): AcceptedMigrationDocumentReport[] { + return initialized.candidateSource.report.documents.flatMap((document) => + document.status === "accepted" ? [structuredClone(document)] : [], + ); +} + +function snapshotFrom(documents: AcceptedMigrationDocumentReport[]): KnowledgeRepositorySnapshot { + return createKnowledgeRepositorySnapshot({ + corpus: initialized.candidateSource.corpus, + creation: CREATION, + documents, + }); +} + +beforeAll(async () => { + canonicalBytesBefore = await Promise.all( + PRIORITY_ONE_SOURCE_PATHS.map((path) => readFile(resolve(REPOSITORY_ROOT, path))), + ); + initialized = await initializeCorpusKnowledgeRepository(options()); +}); + +afterAll(async () => { + await Promise.all( + temporaryDirectories.map((directory) => rm(directory, { force: true, recursive: true })), + ); +}); + +describe("KnowledgeCorpusCandidateSource", () => { + it("loads all eight approved objects with deterministic identity and provenance", async () => { + const batch = await initialized.candidateSource.loadCandidates(); + const ids = batch.candidates.map((candidate) => candidate.metadata.id); + + expect(batch.source).toEqual({ + schemaVersion: "1.0", + sourceId: "founderos-priority-1", + sourceType: "knowledge_corpus", + provenance: { + sourceType: "migration_manifest", + sourceReference: MANIFEST_PATH, + originalCreator: "FounderOS", + }, + }); + expect(initialized.candidateSource.corpus).toMatchObject({ + corpusId: "founderos-priority-1", + corpusVersion: "priority-1-v1", + sourceManifestReference: MANIFEST_PATH, + }); + expect(ids).toHaveLength(8); + expect(ids).toEqual([...ids].sort()); + + for (const candidate of batch.candidates) { + const reportDocument = initialized.candidateSource.report.documents.find( + (document) => document.id === candidate.metadata.id, + ); + expect(reportDocument?.status).toBe("accepted"); + expect(candidate.metadata.source).toEqual({ + sourceType: "official_specification", + sourceReference: reportDocument?.sourcePath, + originalCreator: "FounderOS", + }); + } + }); + + it("rejects an invalid corpus atomically with its migration report", async () => { + const rootPath = await mkdtemp(resolve(tmpdir(), "founderos-corpus-adapter-")); + temporaryDirectories.push(rootPath); + await mkdir(resolve(rootPath, "docs")); + await writeFile(resolve(rootPath, "docs/source.md"), "# Changed source\n", "utf8"); + await writeFile( + resolve(rootPath, "manifest.yaml"), + stringify({ + schemaVersion: "1.0", + corpusId: "invalid-corpus", + documents: [ + { + id: "invalid-object", + objectType: "knowledge", + sourcePath: "docs/source.md", + destinationPath: "knowledge/research/invalid-object.md", + sourceHash: hash("# Expected source\n"), + migrationStatus: "ready", + reviewStatus: "approved", + metadata: { + title: "Invalid object", + domain: "FounderOS", + createdAt: "2026-07-28", + updatedAt: "2026-07-28", + status: "active", + confidence: "high", + importance: "high", + }, + }, + ], + }), + "utf8", + ); + + const error = await KnowledgeCorpusCandidateSource.create({ + rootPath, + manifestPath: "manifest.yaml", + corpusVersion: "invalid-v1", + ...CREATION, + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(KnowledgeCorpusMigrationRejectedError); + expect((error as KnowledgeCorpusMigrationRejectedError).report).toMatchObject({ + corpusId: "invalid-corpus", + status: "rejected", + summary: { rejectedDocuments: 1 }, + documents: [{ status: "rejected", errors: [{ code: "source_hash_mismatch" }] }], + }); + }); + + it("keeps snapshot identity stable across creation metadata and deeply freezes snapshots", async () => { + const second = await KnowledgeCorpusCandidateSource.create( + options({ createdAt: "2030-01-01T00:00:00Z", createdBy: "another-builder" }), + ); + + expect(second.snapshot.snapshotId).toBe(initialized.snapshot.snapshotId); + expect(second.snapshot.contentFingerprint).toBe(initialized.snapshot.contentFingerprint); + expect(second.snapshot.creation).not.toEqual(initialized.snapshot.creation); + expect(Object.isFrozen(initialized.snapshot)).toBe(true); + expect(Object.isFrozen(initialized.snapshot.objects)).toBe(true); + expect(Object.isFrozen(initialized.snapshot.objects[0])).toBe(true); + expect(() => { + initialized.snapshot.objects[0]!.sourceHash = "0".repeat(64); + }).toThrow(TypeError); + }); + + it("queries through the corpus repository with Milestone 05 and 06 compatibility", async () => { + const query = PRIORITY_ONE_QUERY_EVALUATIONS[0]!.query; + const candidates = await initialized.candidateSource.loadCandidates(); + const repositoryResult = await queryKnowledgeRepository(query, initialized.repository); + const candidateArrayResult = queryKnowledgeObjects(query, candidates.candidates); + + expect(serializeKnowledgeQueryResult(repositoryResult)).toBe( + serializeKnowledgeQueryResult(candidateArrayResult), + ); + expect( + (await initialized.repository.getCandidates()).map((object) => object.metadata.id), + ).toEqual(candidates.candidates.map((object) => object.metadata.id)); + }); + + it("does not alter canonical document bytes during initialization", async () => { + const after = await Promise.all( + PRIORITY_ONE_SOURCE_PATHS.map((path) => readFile(resolve(REPOSITORY_ROOT, path))), + ); + expect(after).toEqual(canonicalBytesBefore); + }); +}); + +describe("repository snapshots and change detection", () => { + it("returns an unchanged comparison for identical snapshots", () => { + expect(compareKnowledgeRepositorySnapshots(initialized.snapshot, initialized.snapshot)).toEqual( + { + schemaVersion: "1.0", + previousSnapshotId: initialized.snapshot.snapshotId, + currentSnapshotId: initialized.snapshot.snapshotId, + previousCorpusVersion: "priority-1-v1", + currentCorpusVersion: "priority-1-v1", + corpusVersionChanged: false, + previousContentFingerprint: initialized.snapshot.contentFingerprint, + currentContentFingerprint: initialized.snapshot.contentFingerprint, + contentFingerprintChanged: false, + addedObjectIds: [], + removedObjectIds: [], + identityChanges: [], + sourceHashChanges: [], + metadataChanges: [], + objectChanges: [], + changed: false, + }, + ); + }); + + it("derives metadata, object, and source fingerprints from distinct inputs", () => { + const baseDocuments = acceptedDocuments(); + const base = snapshotFrom(baseDocuments); + const targetId = baseDocuments[0]!.object.metadata.id; + + const metadataDocuments = structuredClone(baseDocuments); + metadataDocuments[0]!.object.metadata.title += " updated"; + const metadataChange = compareKnowledgeRepositorySnapshots( + base, + snapshotFrom(metadataDocuments), + ); + expect(metadataChange.metadataChanges.map((change) => change.objectId)).toEqual([targetId]); + expect(metadataChange.objectChanges.map((change) => change.objectId)).toEqual([targetId]); + expect(metadataChange.sourceHashChanges).toEqual([]); + + const contentDocuments = structuredClone(baseDocuments); + const contentObject = contentDocuments[0]!.object; + if (!("content" in contentObject)) throw new Error("Expected Priority 1 knowledge object"); + contentObject.content += "\nChanged body."; + const contentChange = compareKnowledgeRepositorySnapshots(base, snapshotFrom(contentDocuments)); + expect(contentChange.metadataChanges).toEqual([]); + expect(contentChange.objectChanges.map((change) => change.objectId)).toEqual([targetId]); + expect(contentChange.sourceHashChanges).toEqual([]); + + const sourceDocuments = structuredClone(baseDocuments); + sourceDocuments[0]!.actualSourceHash = "a".repeat(64); + const sourceChange = compareKnowledgeRepositorySnapshots(base, snapshotFrom(sourceDocuments)); + expect(sourceChange.metadataChanges).toEqual([]); + expect(sourceChange.objectChanges).toEqual([]); + expect(sourceChange.sourceHashChanges.map((change) => change.objectId)).toEqual([targetId]); + }); + + it("detects corpus version and canonical-path identity changes", () => { + const current = structuredClone(initialized.snapshot); + const identityRecord = current.objects[0]!; + const previousIdentity = identityRecord.objectId; + identityRecord.objectId = `${previousIdentity}-replacement`; + current.objects.sort((left, right) => left.objectId.localeCompare(right.objectId)); + current.corpusVersion = "priority-1-v2"; + current.snapshotId = `snapshot-${"d".repeat(64)}`; + current.contentFingerprint = "d".repeat(64); + + const changeSet = compareKnowledgeRepositorySnapshots(initialized.snapshot, current); + expect(changeSet).toMatchObject({ + corpusVersionChanged: true, + contentFingerprintChanged: true, + addedObjectIds: [`${previousIdentity}-replacement`], + removedObjectIds: [previousIdentity], + identityChanges: [ + { + sourcePath: identityRecord.sourcePath, + previousObjectId: previousIdentity, + currentObjectId: `${previousIdentity}-replacement`, + }, + ], + changed: true, + }); + }); + + it("detects content identity changes not represented by object fingerprints", () => { + const pathDocuments = acceptedDocuments(); + pathDocuments[0]!.sourcePath = "docs/relocated-source.md"; + const changed = compareKnowledgeRepositorySnapshots( + initialized.snapshot, + snapshotFrom(pathDocuments), + ); + + expect(changed).toMatchObject({ + contentFingerprintChanged: true, + sourceHashChanges: [], + metadataChanges: [], + objectChanges: [], + changed: true, + }); + }); +});