From 9112dcdc6c962bc81c20c9c8320f08bb94c0310e Mon Sep 17 00:00:00 2001 From: SoulSpace Creator Date: Tue, 28 Jul 2026 11:44:59 +0800 Subject: [PATCH] feat: materialize FounderOS core knowledge vault --- .gitignore | 1 + AGENTS.md | 83 ++++ ARCHITECTURE_DECISIONS.md | 8 + CHANGELOG.md | 3 + DOCUMENTATION_INDEX.md | 11 + README.md | 6 +- ...underOS_Knowledge_Corpus_Structure_v1.0.md | 21 + ...e_Migration_Manifest_Specification_v1.0.md | 40 ++ ...dge_Object_Lifecycle_Specification_v1.0.md | 34 ++ ...S_Milestone_04_Acceptance_Criteria_v1.0.md | 70 +++ ...ilestone_04_Codex_Execution_Prompt_v1.0.md | 66 +++ ...ault_Materialization_Specification_v1.0.md | 60 +++ ...ilestone_04_Verification_Checklist_v1.0.md | 92 ++++ docs/reviews/REPOSITORY_AUDIT_v1.0.md | 398 ++++++++++++++++++ knowledge/README.md | 19 + knowledge/migration-manifest.yaml | 163 +++++++ package.json | 1 + packages/knowledge-schema/README.md | 8 + packages/knowledge-schema/src/index.ts | 1 + packages/knowledge-schema/src/migration.ts | 125 ++++++ .../knowledge-schema/tests/migration.test.ts | 96 +++++ services/knowledge-engine/README.md | 8 + services/knowledge-engine/package.json | 1 + .../execute-knowledge-migration.ts | 280 ++++++++++++ .../src/application/run-migration-command.ts | 28 ++ services/knowledge-engine/src/cli/migrate.ts | 51 +++ .../knowledge-engine/src/domain/safe-path.ts | 6 + services/knowledge-engine/src/index.ts | 6 + .../infrastructure/load-migration-manifest.ts | 104 +++++ .../src/infrastructure/safe-path.ts | 109 +++++ .../src/interfaces/migration-report.ts | 72 ++++ .../knowledge-engine/tests/migration.test.ts | 303 +++++++++++++ 32 files changed, 2273 insertions(+), 1 deletion(-) create mode 100644 AGENTS.md create mode 100644 docs/milestones/milestone-04/FounderOS_Knowledge_Corpus_Structure_v1.0.md create mode 100644 docs/milestones/milestone-04/FounderOS_Knowledge_Migration_Manifest_Specification_v1.0.md create mode 100644 docs/milestones/milestone-04/FounderOS_Knowledge_Object_Lifecycle_Specification_v1.0.md create mode 100644 docs/milestones/milestone-04/FounderOS_Milestone_04_Acceptance_Criteria_v1.0.md create mode 100644 docs/milestones/milestone-04/FounderOS_Milestone_04_Codex_Execution_Prompt_v1.0.md create mode 100644 docs/milestones/milestone-04/FounderOS_Milestone_04_Core_Vault_Materialization_Specification_v1.0.md create mode 100644 docs/milestones/milestone-04/FounderOS_Milestone_04_Verification_Checklist_v1.0.md create mode 100644 docs/reviews/REPOSITORY_AUDIT_v1.0.md create mode 100644 knowledge/README.md create mode 100644 knowledge/migration-manifest.yaml create mode 100644 packages/knowledge-schema/src/migration.ts create mode 100644 packages/knowledge-schema/tests/migration.test.ts create mode 100644 services/knowledge-engine/src/application/execute-knowledge-migration.ts create mode 100644 services/knowledge-engine/src/application/run-migration-command.ts create mode 100644 services/knowledge-engine/src/cli/migrate.ts create mode 100644 services/knowledge-engine/src/domain/safe-path.ts create mode 100644 services/knowledge-engine/src/infrastructure/load-migration-manifest.ts create mode 100644 services/knowledge-engine/src/infrastructure/safe-path.ts create mode 100644 services/knowledge-engine/src/interfaces/migration-report.ts create mode 100644 services/knowledge-engine/tests/migration.test.ts diff --git a/.gitignore b/.gitignore index 4e34ae3..dc86ab5 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ coverage/ dist/ node_modules/ *.log +migration-report.json diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..91b6293 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,83 @@ +# FounderOS AGENTS.md + +## Role + +You are an AI engineering agent working inside the FounderOS repository. + +Your responsibility is to implement approved specifications while +preserving architecture integrity, documentation consistency, and +engineering quality. + +## Before Coding + +Always read and understand: + +- README.md +- Relevant architecture documentation +- Relevant milestone specifications +- Existing implementation patterns +- Current tests + +Do not start implementation without understanding repository context. + +## Engineering Principles + +Follow: + +- Documentation first +- Architecture before code +- Small verified changes +- Preserve existing boundaries +- Prefer simple maintainable solutions +- Add tests for behavior changes + +## Implementation Rules + +You should: + +- Follow existing package boundaries +- Maintain TypeScript standards +- Reuse existing patterns +- Update documentation when required + +You should not: + +- Redesign architecture without approval +- Introduce unnecessary dependencies +- Modify unrelated modules +- Ignore specifications + +## Verification Requirements + +Before completion run: + +```bash +pnpm format:check +pnpm lint +pnpm build +pnpm test +``` + +If verification fails: + +- Investigate +- Report clearly +- Do not claim completion + +## Completion Report + +Every completed task must include: + +1. Summary +2. Changed files +3. Tests executed +4. Verification results +5. Risks or limitations +6. Recommended next steps + +## Principle + +Operate as a senior engineer inside an AI-native engineering +organization. + +Preserve system integrity over short-term implementation speed. diff --git a/ARCHITECTURE_DECISIONS.md b/ARCHITECTURE_DECISIONS.md index 3d46d0b..57bfc8b 100644 --- a/ARCHITECTURE_DECISIONS.md +++ b/ARCHITECTURE_DECISIONS.md @@ -58,6 +58,14 @@ This ledger records repository-level decisions. Feature-level decisions should m - **Decision:** Recursively ingest regular Markdown files from one explicit directory in stable relative-path order, never follow symbolic links, and return a deterministic aggregate report. Reject every member of duplicate object-ID or source-hash sets so accepted objects have unique identity and source content. - **Consequences:** Migration reports are reproducible and safe to review before persistence. Directory ingestion remains an explicit, read-only operation rather than a vault crawler, watcher, scheduler, or storage system. +## ADR-0008: Materialize the core corpus through a reviewed manifest + +- **Status:** Accepted +- **Date:** 2026-07-28 +- **Context:** Milestone 03 proved deterministic directory ingestion with derived fixtures, but the complete Priority 1 corpus needs a controlled contract that detects source drift and preserves human review without modifying or duplicating canonical specifications. +- **Decision:** Use a strict, versioned YAML manifest to bind each knowledge object ID and type to one canonical source path, logical `knowledge/` destination, expected SHA-256 digest, schema metadata, migration status, and review status. Execute the manifest within one physical repository root, reject traversal and symbolic links, and materialize validated objects only in a deterministic report artifact. +- **Consequences:** The eight Priority 1 documents become reproducibly auditable without adding persistence or changing canonical Markdown. Manifest hashes must be deliberately updated when approved source documents change, and report content is a normalized schema representation rather than a byte-for-byte source replica. + ## ADR template ```markdown diff --git a/CHANGELOG.md b/CHANGELOG.md index f0ca8b3..ec10f11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,3 +15,6 @@ All notable changes to FounderOS will be documented here. - File-level acceptance and rejection reports with deterministic source evidence. - Deterministic directory-ingestion reports with duplicate identity and source detection. - Canonical templates for all seven knowledge object types and a five-document FounderOS Core migration pilot. +- Strict migration-manifest schemas with identity, path, hash, metadata, lifecycle, and review validation. +- Manifest-controlled migration of all eight FounderOS Priority 1 canonical documents. +- Path-contained migration CLI and deterministic `migration-report.json` generation. diff --git a/DOCUMENTATION_INDEX.md b/DOCUMENTATION_INDEX.md index bca942f..380062c 100644 --- a/DOCUMENTATION_INDEX.md +++ b/DOCUMENTATION_INDEX.md @@ -53,8 +53,19 @@ The documents below are the official FounderOS v1.0 bootstrap specification, org - [Milestone 02 Vault Ingestion Foundation Specification v1.0](./docs/migration/FounderOS_Milestone_02_Vault_Ingestion_Foundation_Specification_v1.0.md) - [Milestone 03 Core Migration Dry Run Specification v1.0](./docs/migration/FounderOS_Milestone_03_Core_Migration_Dry_Run_Specification_v1.0.md) +### Milestone 04 — Core Vault Materialization + +- [Core Vault Materialization Specification v1.0](./docs/milestones/milestone-04/FounderOS_Milestone_04_Core_Vault_Materialization_Specification_v1.0.md) +- [Knowledge Migration Manifest Specification v1.0](./docs/milestones/milestone-04/FounderOS_Knowledge_Migration_Manifest_Specification_v1.0.md) +- [Knowledge Corpus Structure v1.0](./docs/milestones/milestone-04/FounderOS_Knowledge_Corpus_Structure_v1.0.md) +- [Knowledge Object Lifecycle Specification v1.0](./docs/milestones/milestone-04/FounderOS_Knowledge_Object_Lifecycle_Specification_v1.0.md) +- [Milestone 04 Acceptance Criteria v1.0](./docs/milestones/milestone-04/FounderOS_Milestone_04_Acceptance_Criteria_v1.0.md) +- [Milestone 04 Verification Checklist v1.0](./docs/milestones/milestone-04/FounderOS_Milestone_04_Verification_Checklist_v1.0.md) +- [Milestone 04 Codex Execution Prompt v1.0](./docs/milestones/milestone-04/FounderOS_Milestone_04_Codex_Execution_Prompt_v1.0.md) + ## Repository governance - [Architecture decisions](./ARCHITECTURE_DECISIONS.md) - [Contribution guide](./CONTRIBUTING.md) - [Changelog](./CHANGELOG.md) +- [Repository Audit v1.0](./docs/reviews/REPOSITORY_AUDIT_v1.0.md) diff --git a/README.md b/README.md index c1fc11f..f5d160a 100644 --- a/README.md +++ b/README.md @@ -21,8 +21,9 @@ The official specifications are indexed in [DOCUMENTATION_INDEX.md](./DOCUMENTAT ## Implemented foundations - [`@founderos/knowledge-schema`](./packages/knowledge-schema/README.md) provides strict runtime schemas and inferred TypeScript contracts for KnowledgeOS metadata, relationships, and the seven official knowledge object categories. -- [`@founderos/knowledge-engine`](./services/knowledge-engine/README.md) provides read-only file and directory ingestion, deterministic migration reports, conflict detection, and source evidence. +- [`@founderos/knowledge-engine`](./services/knowledge-engine/README.md) provides read-only file and directory ingestion, manifest-controlled Priority 1 corpus migration, deterministic reports, path containment, conflict detection, and source evidence. - [`specs/knowledge-templates`](./specs/knowledge-templates) provides valid Markdown templates for all seven KnowledgeOS object types. +- [`knowledge/migration-manifest.yaml`](./knowledge/migration-manifest.yaml) binds the eight canonical FounderOS Priority 1 documents to reviewed object identities, logical destinations, metadata, and source hashes. Vault watching, persistence, retrieval, embeddings, graph storage, agent behavior, connectors, and interfaces remain unimplemented. @@ -55,6 +56,9 @@ pnpm lint pnpm typecheck pnpm test pnpm build +pnpm knowledge:migrate ``` +`pnpm knowledge:migrate` validates the approved manifest and writes the deterministic, Git-ignored `migration-report.json` artifact without modifying canonical documents. + See [CONTRIBUTING.md](./CONTRIBUTING.md) before making changes. diff --git a/docs/milestones/milestone-04/FounderOS_Knowledge_Corpus_Structure_v1.0.md b/docs/milestones/milestone-04/FounderOS_Knowledge_Corpus_Structure_v1.0.md new file mode 100644 index 0000000..38641a2 --- /dev/null +++ b/docs/milestones/milestone-04/FounderOS_Knowledge_Corpus_Structure_v1.0.md @@ -0,0 +1,21 @@ +# FounderOS Knowledge Corpus Structure v1.0 + +## Purpose + +Define the long-term KnowledgeOS organization structure. + +## Structure + + knowledge/ + + ├── governance/ + ├── architecture/ + ├── decisions/ + ├── projects/ + ├── research/ + ├── experiments/ + └── principles/ + +## Principle + +Knowledge organization should reflect organizational intelligence. diff --git a/docs/milestones/milestone-04/FounderOS_Knowledge_Migration_Manifest_Specification_v1.0.md b/docs/milestones/milestone-04/FounderOS_Knowledge_Migration_Manifest_Specification_v1.0.md new file mode 100644 index 0000000..fbe11bc --- /dev/null +++ b/docs/milestones/milestone-04/FounderOS_Knowledge_Migration_Manifest_Specification_v1.0.md @@ -0,0 +1,40 @@ +# FounderOS Knowledge Migration Manifest Specification v1.0 + +## Purpose + +Define the manifest used to control and audit KnowledgeOS migrations. + +## Manifest Fields + +Each entry should contain: + +- Object ID +- Object type +- Source path +- Destination path +- Source hash +- Review status + +## Example + +``` yaml +documents: + - id: founderos-constitution + type: principle + source: docs/governance/FounderOS_Constitution_v1.0.md + status: approved +``` + +## Validation Rules + +Migration fails when: + +- Source file is missing +- IDs conflict +- Hash validation fails +- Metadata is invalid + +## Principle + +The migration manifest is the audit contract between human knowledge and +machine processing. diff --git a/docs/milestones/milestone-04/FounderOS_Knowledge_Object_Lifecycle_Specification_v1.0.md b/docs/milestones/milestone-04/FounderOS_Knowledge_Object_Lifecycle_Specification_v1.0.md new file mode 100644 index 0000000..7a8139a --- /dev/null +++ b/docs/milestones/milestone-04/FounderOS_Knowledge_Object_Lifecycle_Specification_v1.0.md @@ -0,0 +1,34 @@ +# FounderOS Knowledge Object Lifecycle Specification v1.0 + +## Lifecycle + + Draft + | + v + Validated + | + v + Approved + | + v + Indexed + | + v + Retrieved + | + v + Archived + +## Rules + +Validated objects require: + +- Schema validation +- Metadata validation +- Provenance checks + +Approved objects require human review. + +## Principle + +Knowledge quality improves through controlled lifecycle management. diff --git a/docs/milestones/milestone-04/FounderOS_Milestone_04_Acceptance_Criteria_v1.0.md b/docs/milestones/milestone-04/FounderOS_Milestone_04_Acceptance_Criteria_v1.0.md new file mode 100644 index 0000000..837426a --- /dev/null +++ b/docs/milestones/milestone-04/FounderOS_Milestone_04_Acceptance_Criteria_v1.0.md @@ -0,0 +1,70 @@ +# FounderOS Milestone 04 Acceptance Criteria v1.0 + +## Purpose + +Define the completion criteria for FounderOS Milestone 04 --- Core Vault +Materialization. + +## Functional Acceptance Criteria + +The milestone is complete when: + +- [ ] Migration manifest format is implemented. +- [ ] Manifest validation is implemented. +- [ ] FounderOS Priority 1 knowledge corpus can be migrated. +- [ ] Migration workflow produces deterministic reports. +- [ ] Migration execution does not modify source documents. + +## Migration Criteria + +The migration system must verify: + +- [ ] Every document has a unique knowledge object ID. +- [ ] Every source document exists. +- [ ] Every document type is supported. +- [ ] Source hashes can be verified. +- [ ] Provenance information is preserved. + +## Error Handling Criteria + +The system must fail clearly when: + +- [ ] Duplicate object IDs are detected. +- [ ] Source documents are missing. +- [ ] Metadata validation fails. +- [ ] Manifest entries are invalid. +- [ ] Source content has changed unexpectedly. + +## Engineering Acceptance Criteria + +Required: + +- [ ] TypeScript implementation follows existing conventions. +- [ ] Existing package boundaries are preserved. +- [ ] Tests are added for new behavior. +- [ ] Documentation is updated where necessary. + +## Verification Requirements + +The following commands must pass: + +``` bash +pnpm format:check +pnpm lint +pnpm build +pnpm test +``` + +## Definition of Done + +Milestone 04 is complete when the KnowledgeOS migration process is: + +- Repeatable +- Auditable +- Deterministic +- Safe for future corpus expansion + +## Principle + +A knowledge foundation must be trustworthy before it becomes +intelligent. diff --git a/docs/milestones/milestone-04/FounderOS_Milestone_04_Codex_Execution_Prompt_v1.0.md b/docs/milestones/milestone-04/FounderOS_Milestone_04_Codex_Execution_Prompt_v1.0.md new file mode 100644 index 0000000..14e9924 --- /dev/null +++ b/docs/milestones/milestone-04/FounderOS_Milestone_04_Codex_Execution_Prompt_v1.0.md @@ -0,0 +1,66 @@ +# FounderOS Milestone 04 Codex Execution Prompt v1.0 + +## Role + +You are the lead engineer implementing FounderOS Milestone 04. + +## Context + +FounderOS has completed: + +- Repository foundation +- Knowledge schema foundation +- Markdown ingestion +- Core migration dry run + +## Objective + +Implement: + +- Migration manifest +- Full corpus migration workflow +- Deterministic reports +- CLI migration interface +- Acceptance tests + +## Constraints + +Do not implement: + +- Database persistence +- Embeddings +- Retrieval ranking +- Knowledge graph +- Agents +- MCP integrations + +## Read First + +- README.md +- AGENTS.md +- Repository audit +- Architecture documentation +- Existing knowledge-engine code + +## Verification + +Run: + +- pnpm format:check +- pnpm lint +- pnpm build +- pnpm test + +## Completion Report + +Return: + +1. Summary +2. Changed files +3. Tests executed +4. Risks +5. Next milestone recommendation + +## Principle + +Build a reliable knowledge foundation before adding intelligence layers. diff --git a/docs/milestones/milestone-04/FounderOS_Milestone_04_Core_Vault_Materialization_Specification_v1.0.md b/docs/milestones/milestone-04/FounderOS_Milestone_04_Core_Vault_Materialization_Specification_v1.0.md new file mode 100644 index 0000000..af4afb6 --- /dev/null +++ b/docs/milestones/milestone-04/FounderOS_Milestone_04_Core_Vault_Materialization_Specification_v1.0.md @@ -0,0 +1,60 @@ +# FounderOS Milestone 04 Core Vault Materialization Specification v1.0 + +## Purpose + +Define the transition from Milestone 03 migration dry run into a +complete, controlled, and auditable FounderOS Core Knowledge Vault +migration. + +## Objective + +Create a production-ready KnowledgeOS foundation without introducing +databases, embeddings, retrieval ranking, or agent execution. + +## Scope + +Included: + +- Complete Priority 1 knowledge corpus migration +- Migration manifest +- Canonical source tracking +- Deterministic migration reports +- CLI migration workflow +- Acceptance verification + +Excluded: + +- Vector database +- Embeddings +- Knowledge graph persistence +- Retrieval ranking +- Agent runtime +- MCP integrations + +## Migration Flow + + Canonical Documents + | + v + Migration Manifest + | + v + Knowledge Engine + | + v + Validated Knowledge Objects + | + v + Migration Report + +## Success Criteria + +- All Priority 1 documents migrate successfully. +- Every object has traceable provenance. +- Source documents remain unchanged. +- Migration results are deterministic. + +## Principle + +Knowledge integrity must be established before knowledge intelligence is +built. diff --git a/docs/milestones/milestone-04/FounderOS_Milestone_04_Verification_Checklist_v1.0.md b/docs/milestones/milestone-04/FounderOS_Milestone_04_Verification_Checklist_v1.0.md new file mode 100644 index 0000000..6d3cf86 --- /dev/null +++ b/docs/milestones/milestone-04/FounderOS_Milestone_04_Verification_Checklist_v1.0.md @@ -0,0 +1,92 @@ +# FounderOS Milestone 04 Verification Checklist v1.0 + +## Purpose + +Define the verification process for Milestone 04 --- Core Vault +Materialization. + +------------------------------------------------------------------------ + +# 1. Repository Verification + +Checklist: + +- [ ] Milestone 04 documents exist. +- [ ] New files follow repository documentation conventions. +- [ ] No unrelated files are modified. + +------------------------------------------------------------------------ + +# 2. Migration Workflow Verification + +Checklist: + +- [ ] Migration manifest can be loaded. +- [ ] Valid documents migrate successfully. +- [ ] Invalid documents fail with clear errors. +- [ ] Migration reports are generated. + +------------------------------------------------------------------------ + +# 3. Data Integrity Verification + +Checklist: + +- [ ] Original source documents remain unchanged. +- [ ] Source hashes are recorded. +- [ ] Object IDs are unique. +- [ ] Provenance information is preserved. + +------------------------------------------------------------------------ + +# 4. Determinism Verification + +Checklist: + +- [ ] Same input produces identical output. +- [ ] Document ordering is stable. +- [ ] Report serialization is deterministic. + +------------------------------------------------------------------------ + +# 5. Safety Verification + +Checklist: + +- [ ] Migration cannot access files outside the approved root. +- [ ] Symbolic link behavior is controlled. +- [ ] No uncontrolled filesystem mutation occurs. + +------------------------------------------------------------------------ + +# 6. Engineering Verification + +Checklist: + +- [ ] Format check passes. +- [ ] Lint passes. +- [ ] Build passes. +- [ ] Tests pass. + +Commands: + +``` bash +pnpm format:check +pnpm lint +pnpm build +pnpm test +``` + +------------------------------------------------------------------------ + +# Milestone Approval + +Milestone 04 can be approved when: + +- Functional requirements are complete. +- Verification checklist passes. +- Codex execution report confirms readiness. + +## Principle + +Verification converts implementation into reliable progress. diff --git a/docs/reviews/REPOSITORY_AUDIT_v1.0.md b/docs/reviews/REPOSITORY_AUDIT_v1.0.md new file mode 100644 index 0000000..ee5c1a2 --- /dev/null +++ b/docs/reviews/REPOSITORY_AUDIT_v1.0.md @@ -0,0 +1,398 @@ +# FounderOS Repository Audit v1.0 + +## Audit metadata + +| Field | Value | +| --- | --- | +| Audit date | 2026-07-28 | +| Repository | `adamshen1007/FounderOS` | +| Checked-out branch | `codex/milestone-03` | +| Checked-out commit | `cb09af5` — `feat: add FounderOS core migration dry run` | +| Remote default branch | `origin/main` at `14336e7` | +| Merge status | Milestone 03 is merged into `origin/main` through pull request #1 | +| Audit method | Repository inventory, specification-to-implementation comparison, source/test review, Git history, and CI verification | + +## Executive summary + +FounderOS is a documentation-first TypeScript monorepo with a sound engineering foundation and a deliberately narrow first implementation slice. The repository has completed its base initialization, runtime KnowledgeOS schema contracts, single-file Markdown ingestion, and a deterministic directory migration dry run. The only implemented product-domain code is the KnowledgeOS schema and ingestion foundation; applications, agents, Hermes, memory persistence, MCP, integrations, infrastructure, retrieval, embeddings, and graph persistence remain intentional placeholders or unimplemented capabilities. + +The architecture currently has a clean dependency direction: + +```text +Markdown/frontmatter + | + v +@founderos/knowledge-engine + parse -> normalize -> validate -> report + | + v + @founderos/knowledge-schema +``` + +The next milestone should turn the successful five-document dry run into a controlled, reviewable migration of the complete FounderOS Priority 1 corpus. It should add a migration manifest and executable report workflow while continuing to defer databases, embeddings, retrieval ranking, graph persistence, Hermes, agents, MCP connectors, and UI. + +## 1. Full directory tree + +This is the complete logical project tree at audit time, including the new audit report and intentional empty specification directories. Generated or machine-local content is excluded: `.git/`, `node_modules/`, `dist/`, `.build/`, coverage output, caches, and `.DS_Store`. + +```text +FounderOS/ +├── .editorconfig +├── .github/ +│ └── workflows/ +│ └── ci.yml +├── .gitignore +├── .nvmrc +├── .prettierignore +├── ARCHITECTURE_DECISIONS.md +├── CHANGELOG.md +├── CONTRIBUTING.md +├── DOCUMENTATION_INDEX.md +├── README.md +├── apps/ +│ └── README.md +├── docs/ +│ ├── README.md +│ ├── agents/ +│ │ ├── AI_Task_Routing_Framework_v1.0.md +│ │ ├── Agent_Communication_Protocol_v1.0.md +│ │ ├── Agent_Evaluation_Framework_v1.0.md +│ │ ├── FounderOS_Agent_Runtime_Specification_v1.0.md +│ │ └── Hermes_Chief_of_Staff_Specification_v1.0.md +│ ├── architecture/ +│ │ ├── FounderOS_Data_Architecture_Specification_v1.0.md +│ │ ├── FounderOS_MCP_Architecture_Specification_v1.0.md +│ │ ├── FounderOS_Repository_Architecture_Specification_v1.0.md +│ │ ├── FounderOS_Security_and_Governance_Architecture_Specification_v1.0.md +│ │ └── FounderOS_System_Architecture_Specification_v1.0.md +│ ├── engineering/ +│ │ ├── FounderOS_CI_CD_Architecture_Specification_v1.0.md +│ │ ├── FounderOS_Codex_Execution_Framework_v1.0.md +│ │ ├── FounderOS_Coding_Standards_Specification_v1.0.md +│ │ ├── FounderOS_Engineering_Strategy_Specification_v1.0.md +│ │ ├── FounderOS_Testing_Strategy_Specification_v1.0.md +│ │ └── REPOSITORY_INITIALIZATION_PLAN.md +│ ├── governance/ +│ │ ├── FounderOS_Constitution_v1.0.md +│ │ ├── FounderOS_Decision_Framework_v1.0.md +│ │ └── FounderOS_Design_Principles_v1.0.md +│ ├── knowledgeos/ +│ │ ├── KnowledgeOS_Architecture_Specification_v1.0.md +│ │ ├── KnowledgeOS_Knowledge_Graph_Architecture_Specification_v1.0.md +│ │ ├── KnowledgeOS_Knowledge_Object_Model_Specification_v1.0.md +│ │ ├── KnowledgeOS_Metadata_System_Specification_v1.0.md +│ │ ├── KnowledgeOS_Retrieval_Engine_Specification_v1.0.md +│ │ └── KnowledgeOS_Vault_Architecture_Specification_v1.0.md +│ ├── migration/ +│ │ ├── FounderOS_Codex_First_Execution_Prompt_v1.0.md +│ │ ├── FounderOS_Knowledge_Migration_Strategy_v1.0.md +│ │ ├── FounderOS_Milestone_00_Repository_Initialization_Specification_v1.0.md +│ │ ├── FounderOS_Milestone_02_Vault_Ingestion_Foundation_Specification_v1.0.md +│ │ ├── FounderOS_Milestone_03_Core_Migration_Dry_Run_Specification_v1.0.md +│ │ ├── FounderOS_Vault_Initialization_Specification_v1.0.md +│ │ ├── OpportunityOS_Migration_Specification_v1.0.md +│ │ └── Speculor_AI_Migration_Specification_v1.0.md +│ └── reviews/ +│ └── REPOSITORY_AUDIT_v1.0.md +├── eslint.config.mjs +├── infrastructure/ +│ └── README.md +├── integrations/ +│ └── README.md +├── package.json +├── packages/ +│ ├── README.md +│ ├── agent-contracts/ +│ │ └── README.md +│ ├── knowledge-schema/ +│ │ ├── README.md +│ │ ├── package.json +│ │ ├── src/ +│ │ │ ├── enums.ts +│ │ │ ├── index.ts +│ │ │ ├── metadata.ts +│ │ │ ├── objects.ts +│ │ │ ├── parse.ts +│ │ │ └── primitives.ts +│ │ ├── tests/ +│ │ │ ├── fixtures.ts +│ │ │ ├── metadata.test.ts +│ │ │ └── objects.test.ts +│ │ ├── tsconfig.build.json +│ │ └── tsconfig.json +│ ├── memory-types/ +│ │ └── README.md +│ └── shared-config/ +│ └── README.md +├── pnpm-lock.yaml +├── pnpm-workspace.yaml +├── prettier.config.mjs +├── scripts/ +│ └── repository-foundation.ts +├── services/ +│ ├── README.md +│ ├── agent-router/ +│ │ └── README.md +│ ├── hermes-runtime/ +│ │ └── README.md +│ ├── knowledge-engine/ +│ │ ├── README.md +│ │ ├── package.json +│ │ ├── src/ +│ │ │ ├── application/ +│ │ │ │ ├── ingest-markdown-directory.ts +│ │ │ │ ├── ingest-markdown.ts +│ │ │ │ └── normalize-frontmatter.ts +│ │ │ ├── domain/ +│ │ │ │ └── frontmatter.ts +│ │ │ ├── index.ts +│ │ │ ├── infrastructure/ +│ │ │ │ ├── parse-markdown.ts +│ │ │ │ └── read-markdown-file.ts +│ │ │ └── interfaces/ +│ │ │ ├── directory-ingestion-report.ts +│ │ │ └── ingestion-report.ts +│ │ ├── tests/ +│ │ │ ├── directory-ingestion.test.ts +│ │ │ ├── fixtures/ +│ │ │ │ ├── founderos-constitution.md +│ │ │ │ ├── founderos-core/ +│ │ │ │ │ ├── constitution.md +│ │ │ │ │ ├── decision-framework.md +│ │ │ │ │ ├── design-principles.md +│ │ │ │ │ ├── repository-architecture.md +│ │ │ │ │ └── system-architecture.md +│ │ │ │ ├── invalid-design-principle.md +│ │ │ │ └── reddit-connector-decision.md +│ │ │ ├── ingest-markdown.test.ts +│ │ │ ├── normalize-frontmatter.test.ts +│ │ │ └── parse-markdown.test.ts +│ │ ├── tsconfig.build.json +│ │ └── tsconfig.json +│ ├── mcp-gateway/ +│ │ └── README.md +│ └── memory-service/ +│ └── README.md +├── specs/ +│ ├── README.md +│ ├── agent-protocols/ +│ ├── api/ +│ ├── database/ +│ ├── events/ +│ ├── knowledge-templates/ +│ │ ├── decision.md +│ │ ├── experiment.md +│ │ ├── knowledge.md +│ │ ├── principle.md +│ │ ├── project.md +│ │ ├── relationship.md +│ │ └── research.md +│ └── schemas/ +├── tests/ +│ └── repository-foundation.test.ts +├── tsconfig.build.json +├── tsconfig.json +└── vitest.config.ts +``` + +Inventory totals before this report was added: 112 tracked files, 34 Markdown files under `docs/`, 24 TypeScript source/test/config files under the implementation areas, 7 test files, and 47 declared test cases. Only two workspace units have package manifests. + +## 2. Current applications, packages, and services + +### Applications + +| Component | State | Assessment | +| --- | --- | --- | +| `apps/` | Placeholder | Contains only boundary documentation. No web, mobile, voice, chat, or other UI application exists. This is intentional. | + +### Packages + +| Component | State | Implemented responsibility | +| --- | --- | --- | +| `@founderos/knowledge-schema` | Implemented, version `0.1.0`, private | Strict Zod runtime schemas and inferred TypeScript types for shared metadata, provenance, lifecycle, relationships, and all seven canonical object types: knowledge, decision, project, research, principle, experiment, and relationship. | +| `packages/agent-contracts` | Placeholder | Future agent identity, task, handoff, communication, and evaluation contracts. | +| `packages/memory-types` | Placeholder | Future shared memory contracts; no persistence behavior. | +| `packages/shared-config` | Placeholder | Future validated non-secret configuration contracts. | + +### Services + +| Component | State | Implemented responsibility | +| --- | --- | --- | +| `@founderos/knowledge-engine` | Implemented, version `0.1.0`, private | Read-only YAML-frontmatter Markdown parsing, recursive snake-case-to-camel-case normalization, schema validation, SHA-256 source evidence, file-level reports, deterministic recursive directory ingestion, duplicate ID/hash rejection, and deterministic JSON serialization. | +| `services/agent-router` | Placeholder | No routing or workflow runtime. | +| `services/hermes-runtime` | Placeholder | No Hermes reasoning, context assembly, or conversational runtime. | +| `services/mcp-gateway` | Placeholder | No tool discovery, permission, execution, connector, or audit runtime. | +| `services/memory-service` | Placeholder | No memory persistence or query runtime. | + +### Other architectural areas + +| Area | State | +| --- | --- | +| `integrations/` | Boundary documentation only; no adapters or MCP connectors. | +| `infrastructure/` | Boundary documentation only; no deployment, database, monitoring, or security assets. | +| `specs/knowledge-templates/` | Seven implemented Markdown/frontmatter input templates, one per canonical object type. | +| Other `specs/` domains | Empty placeholders for future APIs, databases, events, schemas, and agent protocols. | +| `scripts/repository-foundation.ts` | Implemented structural verification for required directories/files and Milestone 00 exclusions. | + +## 3. Milestone status + +| Milestone | Status | Evidence and qualification | +| --- | --- | --- | +| Milestone 00 — Repository Initialization | Complete with minor follow-up | Monorepo boundaries, canonical documentation organization, TypeScript, pnpm, ESLint, Prettier, Vitest, CI, repository verification, README, documentation index, and ADR ledger exist. The specification's target tree lists a `LICENSE`, but none is present. | +| Milestone 01 — KnowledgeOS Foundation | Partially delivered and decomposed | The schema, metadata, knowledge-object validation, and ingestion goals are delivered across `@founderos/knowledge-schema` and Milestone 02. A standalone Milestone 01 specification is absent, and the originally listed retrieval interface design has not been implemented. | +| Milestone 02 — Vault Ingestion Foundation | Complete | Single-file Markdown parsing, normalization, schema validation, structured errors, source preservation, real-document-derived fixtures, and acceptance tests are implemented. | +| Milestone 03 — Core Migration Dry Run | Complete and merged | All seven templates, recursive directory ingestion, deterministic aggregate reports, duplicate conflict rejection, symlink containment, and a five-document FounderOS Core pilot are implemented. Commit `cb09af5` is merged to `origin/main` at `14336e7`. | + +No numbered Milestone 04 specification exists yet. + +## 4. Existing architecture + +### Repository architecture + +FounderOS uses a pnpm workspace and strict TypeScript. The intended dependency flow is: + +```text +apps -> services -> packages +agents -> MCP gateway -> integrations -> external systems +infrastructure -> deployment and operations support +``` + +Only `packages/knowledge-schema` and `services/knowledge-engine` are active workspace units. This keeps the current dependency graph small: the knowledge engine depends on the schema package, while the schema package does not depend on services or applications. + +### Knowledge architecture + +The implemented knowledge boundary follows these stages: + +1. Read a caller-selected UTF-8 Markdown file or explicit directory. +2. Parse YAML 1.2 frontmatter and preserve the Markdown body. +3. Normalize keys recursively from specification-style `snake_case` to canonical `camelCase`. +4. Map common identity, classification, provenance, quality, lifecycle, tags, and relationships into the shared metadata envelope. +5. Validate the result against a strict object-specific Zod schema. +6. Return an accepted object or stable, field-addressable rejection errors. +7. Record source path, byte length, and SHA-256 evidence without rewriting source files. +8. For directory batches, sort paths deterministically and reject every member of duplicate object-ID or source-hash sets. + +The human-owned Markdown source remains authoritative. The ingestion layer creates canonical representations and reports, but it does not mutate source, persist objects, crawl continuously, or perform AI extraction. + +### Governance architecture + +- Bootstrap specifications are preserved by domain under `docs/` and indexed from the repository root. +- Repository-level decisions are recorded in a single ADR ledger containing ADR-0001 through ADR-0007. +- The architecture separates reasoning from execution and reserves strategic, external, irreversible, and high-risk actions for human approval. +- Current implementation deliberately stops before agents, tools, external systems, and durable stores, reducing premature security and governance exposure. + +### Engineering and CI architecture + +- Node.js 22+ and pnpm 11+ are declared. +- TypeScript uses strict mode, `noUncheckedIndexedAccess`, consistent casing, and ES modules. +- CI runs frozen dependency installation, formatting, lint, build, type checking, and tests on Ubuntu. +- Workspace packages build before downstream type checking and tests so package export declarations exist in clean CI environments. +- Tests cover repository structure, schema invariants, unsafe/colliding frontmatter, source immutability, deterministic evidence, directory containment, duplicate conflicts, and the five-document pilot. + +## 5. Missing components + +The following components are absent. Most are explicitly deferred rather than defects. + +### KnowledgeOS gaps + +- A production FounderOS vault and the full Priority 1 canonical migration. +- A source-to-object migration manifest or registry. +- An executable migration command that writes reviewable report artifacts. +- Durable database or object-store persistence. +- Embedding generation and vector indexing. +- Retrieval interfaces, filtering, ranking, and result evaluation. +- Knowledge graph persistence, traversal, and relationship intelligence. +- Vault watching, incremental ingestion, background scheduling, and change reconciliation. +- Semantic or AI-assisted extraction for documents without authored frontmatter. + +### Agent and orchestration gaps + +- Hermes runtime. +- Agent runtime, identity enforcement, routing, handoffs, and evaluation execution. +- Memory service and memory-type contracts. +- MCP gateway, permission enforcement, audit execution, and connectors. +- Agent communication/event contracts. + +### Product and platform gaps + +- User-facing web, mobile, chat, or voice applications. +- API contracts and service transport boundaries. +- Authentication, authorization, secrets integration, and runtime policy enforcement. +- Databases, deployment definitions, environments, observability, backup, and disaster recovery. +- External integrations. +- Release packaging, artifact publication, and deployment pipelines. + +### Verification gaps + +- End-to-end and service integration tests. +- Windows/macOS CI for path and filesystem behavior. +- Performance, load, fuzz, and security tests. +- Enforced coverage thresholds. +- Contract compatibility tests for future schema versions. + +## 6. Technical debt and risks + +| Priority | Finding | Impact | Recommended treatment | +| --- | --- | --- | --- | +| High | The Core pilot covers five derived fixtures, but Priority 1 calls for all governance and architecture specifications. Data Architecture, MCP Architecture, and Security/Governance Architecture are not part of the pilot corpus. | The dry run proves the mechanism but not the complete FounderOS Core migration. | Make full-corpus migration the next milestone's primary acceptance target. | +| Medium | Fixture provenance is descriptive rather than mechanically linked to canonical documents. Tests prove fixture bytes are not modified during ingestion, but no manifest binds each fixture to its canonical source path and source checksum. | A derived fixture can drift from the official document without detection. | Add a reviewed source manifest with canonical path, source hash, object ID, and transformation status. | +| Medium | The knowledge engine exposes library functions and a serializer but no supported CLI or repository script for running a batch and writing its report. | Operators must write ad hoc code, weakening repeatability and auditability. | Add a thin, explicit-root migration CLI with deterministic output and no source mutation. | +| Medium | Milestone numbering and scope traceability are incomplete. There is no standalone Milestone 01 specification, while its retrieval-interface item remains outstanding and later work is numbered 02 and 03. | Future contributors may misread what was approved, completed, or intentionally deferred. | Add a milestone ledger and classify the retrieval interface as deferred, superseded, or assigned to a future milestone. | +| Medium | Dependency direction is documented but not mechanically enforced. ESLint uses general recommended rules only. | New packages can accidentally introduce reverse-layer dependencies as the monorepo grows. | Add workspace boundary rules or a dependency-graph check before adding more active services. | +| Medium | Directory reports preserve `normalize(directoryPath)` as `rootPath`. Passing an absolute path therefore makes otherwise deterministic output machine-location-dependent. | Reports are repeatable for the same invocation but may not be byte-identical across workstations. | Define a canonical report root label or require/record a relative logical root separately from the physical input path. | +| Medium | The Milestone 00 target structure includes `LICENSE`, but the repository has no license file and its foundation verifier does not require one. | Usage and contribution rights are undefined. | Select an approved license or explicitly record that the repository is proprietary, then verify the chosen file. | +| Low | Directory ingestion is sequential and retains the complete report/object set in memory. | Large future vaults may ingest slowly or consume excessive memory. | Keep the simple implementation for the next controlled corpus; establish measured limits before optimizing or adding bounded concurrency. | +| Low | CI runs only on Ubuntu, although path normalization and symlink behavior are platform-sensitive. | Windows/macOS regressions may remain undetected. | Add a small cross-platform filesystem test matrix when the ingestion tool becomes operator-facing. | +| Low | Vitest config declares coverage reporters but no coverage command or threshold is enforced in CI. | Test volume can grow while meaningful coverage silently declines. | Add focused package-level thresholds after the next milestone stabilizes the migration surface. | +| Low | Architecture decisions remain in one root ledger rather than immutable individual ADR files. | Review history and supersession will become harder to manage as decisions grow. | Split future decisions into numbered ADR files and retain the root ledger as an index. | + +No `TODO`, `FIXME`, `HACK`, or `XXX` markers were found in tracked project content. The more important debt is capability and governance traceability rather than unfinished inline code. + +## 7. Recommended next milestone + +### Milestone 04 — FounderOS Core Vault Materialization + +#### Objective + +Convert the Milestone 03 dry-run capability into a repeatable, human-reviewed migration of the complete FounderOS Priority 1 corpus while preserving every canonical source document. + +#### Proposed scope + +1. Write `FounderOS_Milestone_04_Core_Vault_Materialization_Specification_v1.0.md` before implementation. +2. Define the approved vault target layout and ownership rules without introducing a database. +3. Add a versioned migration manifest mapping each canonical source path to its intended object ID, object type, derived vault path, source SHA-256, and review status. +4. Add frontmatter-enabled vault copies for all Priority 1 documents: Constitution, Design Principles, Decision Framework, and all five architecture specifications. +5. Add a thin CLI that accepts an explicit root and output path, invokes the existing directory ingestion API, writes deterministic JSON, and never modifies inputs. +6. Add acceptance tests that validate the full corpus, assert unique object IDs/source hashes, detect source/fixture drift, preserve bytes, and reproduce identical report output. +7. Update the documentation index, milestone ledger, README, and ADRs to reflect the operational migration workflow. + +#### Acceptance criteria + +- All eight Priority 1 documents validate successfully as approved KnowledgeOS objects. +- Every migrated object is traceable to one canonical document and reviewed manifest entry. +- Canonical documentation remains byte-for-byte unchanged. +- Two consecutive migration runs produce identical report bytes. +- Invalid, missing, duplicated, or drifted manifest entries fail with actionable errors. +- The CLI cannot crawl outside the explicit root or follow symbolic links. +- Formatting, lint, build, type checking, unit tests, and full-corpus acceptance tests pass in CI. + +#### Continue deferring + +- Databases and object stores. +- Embeddings and vector indexes. +- Retrieval and ranking. +- Knowledge graph persistence. +- Filesystem watching and background scheduling. +- Hermes, agents, MCP integrations, and UI. + +This milestone closes the gap between a test fixture demonstration and an auditable FounderOS knowledge foundation. Retrieval-interface design should follow only after the canonical corpus, identity, provenance, and migration workflow are stable. + +## Verification evidence + +- `git fetch origin main` completed successfully. +- `cb09af5` is an ancestor of `origin/main`; merge commit `14336e7` records pull request #1. +- GitHub Actions run `30320352279` completed successfully for merge commit `14336e7` on 2026-07-28. Install, formatting, lint, build, type check, and test steps all passed. +- Local `pnpm format:check` passed during this audit. +- A local `pnpm lint` attempt produced no output for more than 80 seconds and was interrupted. This is recorded as a local workstation/toolchain condition rather than a repository failure because the same merged revision passed the GitHub Actions lint step. +- Source inventory found 7 test files containing 47 declared test cases. +- No source files were modified as part of this audit; only this report was added. diff --git a/knowledge/README.md b/knowledge/README.md new file mode 100644 index 0000000..c91ac33 --- /dev/null +++ b/knowledge/README.md @@ -0,0 +1,19 @@ +# FounderOS Core knowledge corpus + +This directory defines the logical KnowledgeOS corpus controlled by Milestone 04. The canonical source documents remain under `docs/` and are never rewritten or duplicated here. + +`migration-manifest.yaml` binds every approved Priority 1 object to: + +- its canonical source document; +- its expected SHA-256 source digest; +- its logical destination under `knowledge/governance/` or `knowledge/architecture/`; +- schema-valid metadata; +- migration and human-review status. + +Run the controlled migration from the repository root: + +```bash +pnpm knowledge:migrate +``` + +The command validates and materializes canonical KnowledgeOS objects in `migration-report.json`. The report is deterministic and ignored by Git. No database, index, source mutation, or persistent runtime is introduced. diff --git a/knowledge/migration-manifest.yaml b/knowledge/migration-manifest.yaml new file mode 100644 index 0000000..63e3379 --- /dev/null +++ b/knowledge/migration-manifest.yaml @@ -0,0 +1,163 @@ +schemaVersion: "1.0" +corpusId: founderos-priority-1 +documents: + - id: founderos-constitution-v1 + objectType: knowledge + sourcePath: docs/governance/FounderOS_Constitution_v1.0.md + destinationPath: knowledge/governance/founderos-constitution-v1.md + sourceHash: d51a6b734a4838655b5d56e0e2a309f8845c9b50a3925d2727a3f69a1d13bf7a + migrationStatus: ready + reviewStatus: approved + metadata: + title: FounderOS Constitution v1.0 + domain: FounderOS + category: governance + createdAt: 2026-07-27 + updatedAt: 2026-07-27 + status: active + confidence: high + importance: critical + freshness: current + validationStatus: validated + tags: [FounderOS, constitution, governance] + relationships: [] + - id: founderos-decision-framework-v1 + objectType: knowledge + sourcePath: docs/governance/FounderOS_Decision_Framework_v1.0.md + destinationPath: knowledge/governance/founderos-decision-framework-v1.md + sourceHash: c9e36e4dd1cccb6edb67d55d2d48097e3689592555289641ceff67f2d965fd35 + migrationStatus: ready + reviewStatus: approved + metadata: + title: FounderOS Decision Framework v1.0 + domain: FounderOS + category: governance + createdAt: 2026-07-27 + updatedAt: 2026-07-27 + status: active + confidence: high + importance: critical + freshness: current + validationStatus: validated + tags: [FounderOS, decisions, governance] + relationships: [] + - id: founderos-design-principles-v1 + objectType: knowledge + sourcePath: docs/governance/FounderOS_Design_Principles_v1.0.md + destinationPath: knowledge/governance/founderos-design-principles-v1.md + sourceHash: 06da2bb91c072ec9a310582a7447ec8113bff362e62f199ba73e93468b6a4b14 + migrationStatus: ready + reviewStatus: approved + metadata: + title: FounderOS Design Principles v1.0 + domain: FounderOS + category: governance + createdAt: 2026-07-27 + updatedAt: 2026-07-27 + status: active + confidence: high + importance: critical + freshness: current + validationStatus: validated + tags: [FounderOS, design-principles, governance] + relationships: [] + - id: founderos-data-architecture-v1 + objectType: knowledge + sourcePath: docs/architecture/FounderOS_Data_Architecture_Specification_v1.0.md + destinationPath: knowledge/architecture/founderos-data-architecture-v1.md + sourceHash: dd125f3a0a64deff27fba45c74d96925c649bc46d06bc03b410a417c8614e591 + migrationStatus: ready + reviewStatus: approved + metadata: + title: FounderOS Data Architecture Specification v1.0 + domain: FounderOS + category: architecture + createdAt: 2026-07-27 + updatedAt: 2026-07-27 + status: active + confidence: high + importance: critical + freshness: current + validationStatus: validated + tags: [FounderOS, architecture, data] + relationships: [] + - id: founderos-mcp-architecture-v1 + objectType: knowledge + sourcePath: docs/architecture/FounderOS_MCP_Architecture_Specification_v1.0.md + destinationPath: knowledge/architecture/founderos-mcp-architecture-v1.md + sourceHash: 924833e968d00194868eef1ee660fe0907b60b421de1a24238b98e96f34d3a21 + migrationStatus: ready + reviewStatus: approved + metadata: + title: FounderOS MCP Architecture Specification v1.0 + domain: FounderOS + category: architecture + createdAt: 2026-07-27 + updatedAt: 2026-07-27 + status: active + confidence: high + importance: critical + freshness: current + validationStatus: validated + tags: [FounderOS, architecture, MCP] + relationships: [] + - id: founderos-repository-architecture-v1 + objectType: knowledge + sourcePath: docs/architecture/FounderOS_Repository_Architecture_Specification_v1.0.md + destinationPath: knowledge/architecture/founderos-repository-architecture-v1.md + sourceHash: 68f91dcb5ae9eef684d71be6ea5427b40f42905dde0ecf83c371aa57f6984253 + migrationStatus: ready + reviewStatus: approved + metadata: + title: FounderOS Repository Architecture Specification v1.0 + domain: FounderOS + category: architecture + createdAt: 2026-07-27 + updatedAt: 2026-07-27 + status: active + confidence: high + importance: critical + freshness: current + validationStatus: validated + tags: [FounderOS, architecture, repository] + relationships: [] + - id: founderos-security-governance-architecture-v1 + objectType: knowledge + sourcePath: docs/architecture/FounderOS_Security_and_Governance_Architecture_Specification_v1.0.md + destinationPath: knowledge/architecture/founderos-security-governance-architecture-v1.md + sourceHash: 85f6b5d6e703e298fee49873b51d94bcbe2b01a9d83d42eb0557f975ea382658 + migrationStatus: ready + reviewStatus: approved + metadata: + title: FounderOS Security and Governance Architecture Specification v1.0 + domain: FounderOS + category: architecture + createdAt: 2026-07-27 + updatedAt: 2026-07-27 + status: active + confidence: high + importance: critical + freshness: current + validationStatus: validated + tags: [FounderOS, architecture, governance, security] + relationships: [] + - id: founderos-system-architecture-v1 + objectType: knowledge + sourcePath: docs/architecture/FounderOS_System_Architecture_Specification_v1.0.md + destinationPath: knowledge/architecture/founderos-system-architecture-v1.md + sourceHash: 75766e04bb32c55d964dc88c461937486eb55464c70abfc76b47ce767f80d93d + migrationStatus: ready + reviewStatus: approved + metadata: + title: FounderOS System Architecture Specification v1.0 + domain: FounderOS + category: architecture + createdAt: 2026-07-27 + updatedAt: 2026-07-27 + status: active + confidence: high + importance: critical + freshness: current + validationStatus: validated + tags: [FounderOS, architecture, system] + relationships: [] diff --git a/package.json b/package.json index 7f8600e..091e534 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "format": "prettier --write .", "format:check": "prettier --check .", "lint": "eslint .", + "knowledge:migrate": "pnpm --filter @founderos/knowledge-schema build && pnpm --filter @founderos/knowledge-engine build && node services/knowledge-engine/dist/cli/migrate.js --root . --manifest knowledge/migration-manifest.yaml --output migration-report.json", "test": "pnpm --recursive --if-present run test && vitest run", "test:watch": "vitest", "typecheck": "pnpm --recursive --if-present run typecheck && tsc -p tsconfig.json --noEmit" diff --git a/packages/knowledge-schema/README.md b/packages/knowledge-schema/README.md index 56ed895..e8c2ad2 100644 --- a/packages/knowledge-schema/README.md +++ b/packages/knowledge-schema/README.md @@ -20,6 +20,8 @@ Persistent lifecycle states use `draft`, `review`, `active`, `archived`, and `de The package intentionally contains no persistence, Markdown parsing, retrieval, embedding, graph database, or agent behavior. +Milestone 04 adds strict migration-manifest contracts for object identity, object type, canonical and logical destination paths, SHA-256 evidence, metadata, migration status, and human-review status. These contracts validate migration intent only; filesystem execution remains in `@founderos/knowledge-engine`. + ## Usage ```typescript @@ -29,4 +31,10 @@ const result = KnowledgeObjectSchema.safeParse(input); const knowledgeObject = parseKnowledgeObject(input); ``` +```typescript +import { KnowledgeMigrationManifestSchema } from "@founderos/knowledge-schema"; + +const manifest = KnowledgeMigrationManifestSchema.parse(input); +``` + All schemas reject unknown fields so contract changes remain explicit and versioned. diff --git a/packages/knowledge-schema/src/index.ts b/packages/knowledge-schema/src/index.ts index 9a53047..bbbfe13 100644 --- a/packages/knowledge-schema/src/index.ts +++ b/packages/knowledge-schema/src/index.ts @@ -1,5 +1,6 @@ export * from "./enums.js"; export * from "./metadata.js"; +export * from "./migration.js"; export * from "./objects.js"; export * from "./parse.js"; export * from "./primitives.js"; diff --git a/packages/knowledge-schema/src/migration.ts b/packages/knowledge-schema/src/migration.ts new file mode 100644 index 0000000..065b4f1 --- /dev/null +++ b/packages/knowledge-schema/src/migration.ts @@ -0,0 +1,125 @@ +import { z } from "zod"; + +import { + ConfidenceSchema, + FreshnessSchema, + ImportanceSchema, + KnowledgeObjectTypeSchema, + KnowledgeStatusSchema, +} from "./enums.js"; +import { RelationshipReferenceSchema } from "./metadata.js"; +import { IdentifierSchema, IsoTemporalSchema, NonEmptyStringSchema } from "./primitives.js"; + +const SHA256_PATTERN = /^[a-f0-9]{64}$/u; + +function isSafeRelativePath(value: string): boolean { + if (value.startsWith("/") || value.includes("\\") || value.includes("\0")) { + return false; + } + + const segments = value.split("/"); + return ( + segments.length > 0 && + segments.every((segment) => segment !== "" && segment !== "." && segment !== "..") + ); +} + +export const MigrationStatusSchema = z.enum(["pending", "ready", "migrated", "failed"]); +export const ReviewStatusSchema = z.enum(["pending", "approved", "rejected"]); + +export const MigrationPathSchema = NonEmptyStringSchema.refine( + isSafeRelativePath, + "Expected a normalized relative path without traversal or backslashes", +); + +export const MigrationSourcePathSchema = MigrationPathSchema.refine( + (value) => value.toLowerCase().endsWith(".md"), + "Migration sources must be Markdown files", +); + +export const MigrationDestinationPathSchema = MigrationPathSchema.refine( + (value) => value.startsWith("knowledge/") && value.toLowerCase().endsWith(".md"), + "Migration destinations must be Markdown paths below knowledge/", +); + +export const MigrationMetadataSchema = z + .object({ + title: NonEmptyStringSchema, + domain: NonEmptyStringSchema, + category: NonEmptyStringSchema.optional(), + subCategory: NonEmptyStringSchema.optional(), + createdAt: IsoTemporalSchema, + updatedAt: IsoTemporalSchema, + status: KnowledgeStatusSchema, + confidence: ConfidenceSchema, + importance: ImportanceSchema, + freshness: FreshnessSchema.optional(), + validationStatus: NonEmptyStringSchema.optional(), + tags: z.array(NonEmptyStringSchema).default([]), + relationships: z.array(RelationshipReferenceSchema).default([]), + }) + .strict(); + +export const KnowledgeMigrationManifestEntrySchema = z + .object({ + id: IdentifierSchema, + objectType: KnowledgeObjectTypeSchema, + sourcePath: MigrationSourcePathSchema, + destinationPath: MigrationDestinationPathSchema, + sourceHash: z.string().regex(SHA256_PATTERN, "Expected a lowercase SHA-256 digest"), + migrationStatus: MigrationStatusSchema, + reviewStatus: ReviewStatusSchema, + metadata: MigrationMetadataSchema, + objectData: z.record(z.string(), z.unknown()).default({}), + }) + .strict(); + +export const KnowledgeMigrationManifestSchema = z + .object({ + schemaVersion: z.literal("1.0"), + corpusId: IdentifierSchema, + documents: z.array(KnowledgeMigrationManifestEntrySchema).min(1), + }) + .strict() + .superRefine((manifest, context) => { + const idIndexes = new Map(); + const destinationIndexes = new Map(); + + manifest.documents.forEach((document, index) => { + idIndexes.set(document.id, [...(idIndexes.get(document.id) ?? []), index]); + destinationIndexes.set(document.destinationPath, [ + ...(destinationIndexes.get(document.destinationPath) ?? []), + index, + ]); + }); + + for (const [id, indexes] of idIndexes) { + if (indexes.length > 1) { + for (const index of indexes) { + context.addIssue({ + code: "custom", + message: `Knowledge object ID ${id} is duplicated in the manifest`, + path: ["documents", index, "id"], + }); + } + } + } + + for (const [destinationPath, indexes] of destinationIndexes) { + if (indexes.length > 1) { + for (const index of indexes) { + context.addIssue({ + code: "custom", + message: `Destination path ${destinationPath} is duplicated in the manifest`, + path: ["documents", index, "destinationPath"], + }); + } + } + } + }); + +export type KnowledgeMigrationManifest = z.infer; +export type KnowledgeMigrationManifestEntry = z.infer; +export type MigrationMetadata = z.infer; +export type MigrationStatus = z.infer; +export type ReviewStatus = z.infer; diff --git a/packages/knowledge-schema/tests/migration.test.ts b/packages/knowledge-schema/tests/migration.test.ts new file mode 100644 index 0000000..42ce212 --- /dev/null +++ b/packages/knowledge-schema/tests/migration.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, it } from "vitest"; + +import { KnowledgeMigrationManifestSchema } from "../src/index.js"; + +function validManifest() { + return { + schemaVersion: "1.0", + corpusId: "founderos-priority-1", + documents: [ + { + id: "founderos-constitution-v1", + objectType: "knowledge", + sourcePath: "docs/governance/FounderOS_Constitution_v1.0.md", + destinationPath: "knowledge/governance/founderos-constitution-v1.md", + sourceHash: "a".repeat(64), + migrationStatus: "ready", + reviewStatus: "approved", + metadata: { + title: "FounderOS Constitution v1.0", + domain: "FounderOS", + createdAt: "2026-07-27", + updatedAt: "2026-07-27", + status: "active", + confidence: "high", + importance: "critical", + }, + }, + ], + }; +} + +describe("KnowledgeMigrationManifestSchema", () => { + it("parses a strict, reviewed migration contract", () => { + const manifest = KnowledgeMigrationManifestSchema.parse(validManifest()); + + expect(manifest.documents[0]).toMatchObject({ + id: "founderos-constitution-v1", + migrationStatus: "ready", + objectData: {}, + reviewStatus: "approved", + }); + expect(manifest.documents[0]?.metadata.tags).toEqual([]); + }); + + it("rejects every duplicate object ID", () => { + const input = validManifest(); + input.documents.push({ + ...input.documents[0]!, + destinationPath: "knowledge/governance/duplicate.md", + }); + + const result = KnowledgeMigrationManifestSchema.safeParse(input); + + expect(result.success).toBe(false); + if (!result.success) { + expect(result.error.issues.map((issue) => issue.path.join("."))).toEqual([ + "documents.0.id", + "documents.1.id", + ]); + } + }); + + it.each([ + ["invalid object type", { objectType: "architecture" }], + [ + "invalid metadata", + { metadata: { ...validManifest().documents[0]!.metadata, status: "validated" } }, + ], + ["invalid hash", { sourceHash: "not-a-hash" }], + ["unsafe source path", { sourcePath: "../outside.md" }], + ["unsafe destination path", { destinationPath: "/tmp/object.md" }], + ])("rejects %s", (_label, replacement) => { + const input = validManifest(); + input.documents[0] = { ...input.documents[0]!, ...replacement }; + + expect(KnowledgeMigrationManifestSchema.safeParse(input).success).toBe(false); + }); + + it("rejects duplicate logical destination paths", () => { + const input = validManifest(); + input.documents.push({ + ...input.documents[0]!, + id: "founderos-design-principles-v1", + }); + + const result = KnowledgeMigrationManifestSchema.safeParse(input); + + expect(result.success).toBe(false); + if (!result.success) { + expect(result.error.issues.map((issue) => issue.path.join("."))).toEqual([ + "documents.0.destinationPath", + "documents.1.destinationPath", + ]); + } + }); +}); diff --git a/services/knowledge-engine/README.md b/services/knowledge-engine/README.md index 398b760..e311e54 100644 --- a/services/knowledge-engine/README.md +++ b/services/knowledge-engine/README.md @@ -2,4 +2,12 @@ The Milestone 02 and 03 foundations read one Markdown file or one explicitly selected directory, parse YAML frontmatter, normalize specification-style keys, validate through `@founderos/knowledge-schema`, and return deterministic file-level and aggregate migration reports. +Milestone 04 adds manifest-controlled corpus execution. It loads a strict YAML manifest, confines every read to one physical root, rejects symbolic links and unsafe paths, verifies canonical SHA-256 digests, enforces ready/approved lifecycle gates, creates schema-valid objects from canonical document content, and writes a deterministic report artifact. + +From the repository root: + +```bash +pnpm knowledge:migrate +``` + Directory ingestion is recursive, Markdown-only, stable in path order, and does not follow symbolic links. The implementation remains read-only and does not watch a vault or implement persistence, embeddings, retrieval, graph storage, Hermes, agents, or MCP integrations. diff --git a/services/knowledge-engine/package.json b/services/knowledge-engine/package.json index c814818..95c1f45 100644 --- a/services/knowledge-engine/package.json +++ b/services/knowledge-engine/package.json @@ -17,6 +17,7 @@ }, "scripts": { "build": "tsc -p tsconfig.build.json", + "migrate": "node dist/cli/migrate.js", "test": "vitest run", "typecheck": "tsc -p tsconfig.json --noEmit" }, diff --git a/services/knowledge-engine/src/application/execute-knowledge-migration.ts b/services/knowledge-engine/src/application/execute-knowledge-migration.ts new file mode 100644 index 0000000..f3d0855 --- /dev/null +++ b/services/knowledge-engine/src/application/execute-knowledge-migration.ts @@ -0,0 +1,280 @@ +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; + +import { + KnowledgeObjectSchema, + type KnowledgeMigrationManifestEntry, + type KnowledgeObjectType, +} from "@founderos/knowledge-schema"; + +import { SafePathError } from "../domain/safe-path.js"; +import { loadMigrationManifest } from "../infrastructure/load-migration-manifest.js"; +import { resolvePhysicalRoot, resolveSafeExistingFile } from "../infrastructure/safe-path.js"; +import { + KNOWLEDGE_MIGRATION_REPORT_VERSION, + type KnowledgeMigrationReport, + type KnowledgeMigrationSummary, + type MigrationDocumentReport, + type MigrationError, +} from "../interfaces/migration-report.js"; + +const OBJECT_TYPES: KnowledgeObjectType[] = [ + "decision", + "experiment", + "knowledge", + "principle", + "project", + "relationship", + "research", +]; + +function compareStrings(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +function emptySummary(): KnowledgeMigrationSummary { + return { + acceptedDocuments: 0, + byObjectType: Object.fromEntries(OBJECT_TYPES.map((type) => [type, 0])) as Record< + KnowledgeObjectType, + number + >, + rejectedDocuments: 0, + totalDocuments: 0, + }; +} + +function summarize(documents: MigrationDocumentReport[]): KnowledgeMigrationSummary { + const summary = emptySummary(); + summary.totalDocuments = documents.length; + + for (const document of documents) { + if (document.status === "accepted") { + summary.acceptedDocuments += 1; + summary.byObjectType[document.objectType] += 1; + } else { + summary.rejectedDocuments += 1; + } + } + + return summary; +} + +function reportFailure(manifestPath: string, errors: MigrationError[]): KnowledgeMigrationReport { + return { + corpusId: null, + documents: [], + errors, + manifestPath, + schemaVersion: KNOWLEDGE_MIGRATION_REPORT_VERSION, + status: "rejected", + summary: emptySummary(), + }; +} + +function reportBase(entry: KnowledgeMigrationManifestEntry) { + return { + destinationPath: entry.destinationPath, + expectedSourceHash: entry.sourceHash, + id: entry.id, + migrationStatus: entry.migrationStatus, + objectType: entry.objectType, + reviewStatus: entry.reviewStatus, + sourcePath: entry.sourcePath, + }; +} + +function validationErrors(error: { + issues: ReadonlyArray<{ message: string; path: ReadonlyArray }>; +}): MigrationError[] { + return error.issues.map((issue) => ({ + code: "knowledge_validation_error", + fieldPath: issue.path.length > 0 ? issue.path.map(String).join(".") : "$", + message: issue.message, + })); +} + +function isMissingFileError(error: unknown): boolean { + return error instanceof Error && "code" in error && error.code === "ENOENT"; +} + +async function migrateDocument( + physicalRoot: string, + entry: KnowledgeMigrationManifestEntry, +): Promise { + const base = reportBase(entry); + + if (entry.migrationStatus !== "ready") { + return { + ...base, + errors: [ + { + code: "migration_status_not_ready", + fieldPath: "migrationStatus", + message: `Migration status must be ready, received ${entry.migrationStatus}`, + }, + ], + status: "rejected", + }; + } + + if (entry.reviewStatus !== "approved") { + return { + ...base, + errors: [ + { + code: "review_not_approved", + fieldPath: "reviewStatus", + message: `Review status must be approved, received ${entry.reviewStatus}`, + }, + ], + status: "rejected", + }; + } + + let resolvedSourcePath: string; + try { + resolvedSourcePath = await resolveSafeExistingFile(physicalRoot, entry.sourcePath); + } catch (error: unknown) { + return { + ...base, + errors: [ + { + code: isMissingFileError(error) + ? "source_missing" + : error instanceof SafePathError + ? "source_path_unsafe" + : "source_read_error", + message: isMissingFileError(error) + ? `Source document does not exist: ${entry.sourcePath}` + : error instanceof SafePathError + ? error.message + : `Unable to read source document: ${entry.sourcePath}`, + }, + ], + status: "rejected", + }; + } + + let source: Buffer; + try { + source = await readFile(resolvedSourcePath); + } catch { + return { + ...base, + errors: [ + { + code: "source_read_error", + message: `Unable to read source document: ${entry.sourcePath}`, + }, + ], + status: "rejected", + }; + } + + const actualSourceHash = createHash("sha256").update(source).digest("hex"); + const byteLength = source.byteLength; + + if (actualSourceHash !== entry.sourceHash) { + return { + ...base, + actualSourceHash, + byteLength, + errors: [ + { + code: "source_hash_mismatch", + fieldPath: "sourceHash", + message: `Expected ${entry.sourceHash} but found ${actualSourceHash}`, + }, + ], + status: "rejected", + }; + } + + const candidate = { + ...entry.objectData, + metadata: { + ...entry.metadata, + id: entry.id, + objectType: entry.objectType, + source: { + originalCreator: "FounderOS", + sourceReference: entry.sourcePath, + sourceType: "official_specification", + }, + }, + ...(entry.objectType === "knowledge" ? { content: source.toString("utf8") } : {}), + }; + const validation = KnowledgeObjectSchema.safeParse(candidate); + + if (!validation.success) { + return { + ...base, + actualSourceHash, + byteLength, + errors: validationErrors(validation.error), + status: "rejected", + }; + } + + return { + ...base, + actualSourceHash, + byteLength, + object: validation.data, + status: "accepted", + }; +} + +export interface ExecuteKnowledgeMigrationOptions { + manifestPath: string; + rootPath: string; +} + +export async function executeKnowledgeMigration( + options: ExecuteKnowledgeMigrationOptions, +): Promise { + let physicalRoot: string; + try { + physicalRoot = await resolvePhysicalRoot(options.rootPath); + } catch (error: unknown) { + return reportFailure(options.manifestPath, [ + { + code: "root_path_error", + message: + error instanceof SafePathError + ? error.message + : "Unable to access the approved migration root", + }, + ]); + } + + const loaded = await loadMigrationManifest(physicalRoot, options.manifestPath); + if (loaded.status === "rejected") { + return reportFailure(options.manifestPath, loaded.errors); + } + + const entries = [...loaded.manifest.documents].sort((left, right) => + compareStrings(left.sourcePath, right.sourcePath), + ); + const documents: MigrationDocumentReport[] = []; + + for (const entry of entries) { + documents.push(await migrateDocument(physicalRoot, entry)); + } + + const summary = summarize(documents); + return { + corpusId: loaded.manifest.corpusId, + documents, + errors: [], + manifestPath: options.manifestPath, + schemaVersion: KNOWLEDGE_MIGRATION_REPORT_VERSION, + status: summary.rejectedDocuments === 0 ? "accepted" : "rejected", + summary, + }; +} + +export function serializeKnowledgeMigrationReport(report: KnowledgeMigrationReport): string { + return `${JSON.stringify(report, null, 2)}\n`; +} diff --git a/services/knowledge-engine/src/application/run-migration-command.ts b/services/knowledge-engine/src/application/run-migration-command.ts new file mode 100644 index 0000000..5794e0a --- /dev/null +++ b/services/knowledge-engine/src/application/run-migration-command.ts @@ -0,0 +1,28 @@ +import { writeFile } from "node:fs/promises"; + +import type { KnowledgeMigrationReport } from "../interfaces/migration-report.js"; +import { resolvePhysicalRoot, resolveSafeOutputPath } from "../infrastructure/safe-path.js"; +import { + executeKnowledgeMigration, + serializeKnowledgeMigrationReport, +} from "./execute-knowledge-migration.js"; + +export interface RunMigrationCommandOptions { + manifestPath: string; + outputPath: string; + rootPath: string; +} + +export async function runMigrationCommand( + options: RunMigrationCommandOptions, +): Promise { + const physicalRoot = await resolvePhysicalRoot(options.rootPath); + const outputPath = await resolveSafeOutputPath(physicalRoot, options.outputPath); + const report = await executeKnowledgeMigration({ + manifestPath: options.manifestPath, + rootPath: physicalRoot, + }); + + await writeFile(outputPath, serializeKnowledgeMigrationReport(report), "utf8"); + return report; +} diff --git a/services/knowledge-engine/src/cli/migrate.ts b/services/knowledge-engine/src/cli/migrate.ts new file mode 100644 index 0000000..d0e0ed3 --- /dev/null +++ b/services/knowledge-engine/src/cli/migrate.ts @@ -0,0 +1,51 @@ +import { runMigrationCommand } from "../application/run-migration-command.js"; + +interface MigrationArguments { + manifestPath: string; + outputPath: string; + rootPath: string; +} + +function parseArguments(arguments_: string[]): MigrationArguments { + const normalizedArguments = arguments_[0] === "--" ? arguments_.slice(1) : arguments_; + const values = new Map(); + + for (let index = 0; index < normalizedArguments.length; index += 2) { + const key = normalizedArguments[index]; + const value = normalizedArguments[index + 1]; + if (key === undefined || value === undefined || !key.startsWith("--")) { + throw new Error( + "Usage: migrate --root --manifest --output ", + ); + } + values.set(key, value); + } + + const rootPath = values.get("--root"); + const manifestPath = values.get("--manifest"); + const outputPath = values.get("--output"); + if (rootPath === undefined || manifestPath === undefined || outputPath === undefined) { + throw new Error( + "Usage: migrate --root --manifest --output ", + ); + } + + return { manifestPath, outputPath, rootPath }; +} + +async function main(): Promise { + const arguments_ = parseArguments(process.argv.slice(2)); + const report = await runMigrationCommand(arguments_); + + process.stdout.write( + `Migration ${report.status}: ${report.summary.acceptedDocuments} accepted, ${report.summary.rejectedDocuments} rejected. Report: ${arguments_.outputPath}\n`, + ); + if (report.status === "rejected") { + process.exitCode = 1; + } +} + +main().catch((error: unknown) => { + process.stderr.write(`${error instanceof Error ? error.message : "Migration failed"}\n`); + process.exitCode = 1; +}); diff --git a/services/knowledge-engine/src/domain/safe-path.ts b/services/knowledge-engine/src/domain/safe-path.ts new file mode 100644 index 0000000..4df725e --- /dev/null +++ b/services/knowledge-engine/src/domain/safe-path.ts @@ -0,0 +1,6 @@ +export class SafePathError extends Error { + public constructor(message: string) { + super(message); + this.name = "SafePathError"; + } +} diff --git a/services/knowledge-engine/src/index.ts b/services/knowledge-engine/src/index.ts index ea8bd56..09cec3f 100644 --- a/services/knowledge-engine/src/index.ts +++ b/services/knowledge-engine/src/index.ts @@ -1,7 +1,13 @@ export * from "./application/ingest-markdown.js"; export * from "./application/ingest-markdown-directory.js"; +export * from "./application/execute-knowledge-migration.js"; export * from "./application/normalize-frontmatter.js"; +export * from "./application/run-migration-command.js"; export * from "./domain/frontmatter.js"; +export * from "./domain/safe-path.js"; +export * from "./infrastructure/load-migration-manifest.js"; export * from "./infrastructure/parse-markdown.js"; +export * from "./infrastructure/safe-path.js"; export * from "./interfaces/ingestion-report.js"; export * from "./interfaces/directory-ingestion-report.js"; +export * from "./interfaces/migration-report.js"; diff --git a/services/knowledge-engine/src/infrastructure/load-migration-manifest.ts b/services/knowledge-engine/src/infrastructure/load-migration-manifest.ts new file mode 100644 index 0000000..249b938 --- /dev/null +++ b/services/knowledge-engine/src/infrastructure/load-migration-manifest.ts @@ -0,0 +1,104 @@ +import { readFile } from "node:fs/promises"; + +import { + KnowledgeMigrationManifestSchema, + type KnowledgeMigrationManifest, +} from "@founderos/knowledge-schema"; +import { parseDocument } from "yaml"; + +import { SafePathError } from "../domain/safe-path.js"; +import type { MigrationError } from "../interfaces/migration-report.js"; +import { resolveSafeExistingFile } from "./safe-path.js"; + +export type ManifestLoadResult = + | { manifest: KnowledgeMigrationManifest; status: "accepted" } + | { errors: MigrationError[]; status: "rejected" }; + +function validationErrors(error: { + issues: ReadonlyArray<{ message: string; path: ReadonlyArray }>; +}): MigrationError[] { + return error.issues.map((issue) => ({ + code: "manifest_validation_error", + fieldPath: issue.path.length > 0 ? issue.path.map(String).join(".") : "$", + message: issue.message, + })); +} + +export async function loadMigrationManifest( + physicalRoot: string, + manifestPath: string, +): Promise { + let resolvedManifestPath: string; + + try { + resolvedManifestPath = await resolveSafeExistingFile(physicalRoot, manifestPath); + } catch (error: unknown) { + return { + errors: [ + { + code: error instanceof SafePathError ? "manifest_path_unsafe" : "manifest_read_error", + message: + error instanceof SafePathError + ? error.message + : `Unable to read migration manifest: ${manifestPath}`, + }, + ], + status: "rejected", + }; + } + + let source: string; + try { + source = await readFile(resolvedManifestPath, "utf8"); + } catch { + return { + errors: [ + { + code: "manifest_read_error", + message: `Unable to read migration manifest: ${manifestPath}`, + }, + ], + status: "rejected", + }; + } + + const document = parseDocument(source, { + logLevel: "silent", + prettyErrors: false, + schema: "core", + strict: true, + version: "1.2", + }); + + if (document.errors.length > 0) { + return { + errors: [ + { + code: "manifest_parse_error", + message: document.errors.map((error) => error.message).join("; "), + }, + ], + status: "rejected", + }; + } + + let input: unknown; + try { + input = document.toJS({ maxAliasCount: 20 }) as unknown; + } catch (error: unknown) { + return { + errors: [ + { + code: "manifest_parse_error", + message: error instanceof Error ? error.message : "Unable to parse migration manifest", + }, + ], + status: "rejected", + }; + } + + const result = KnowledgeMigrationManifestSchema.safeParse(input); + return result.success + ? { manifest: result.data, status: "accepted" } + : { errors: validationErrors(result.error), status: "rejected" }; +} diff --git a/services/knowledge-engine/src/infrastructure/safe-path.ts b/services/knowledge-engine/src/infrastructure/safe-path.ts new file mode 100644 index 0000000..b8096d5 --- /dev/null +++ b/services/knowledge-engine/src/infrastructure/safe-path.ts @@ -0,0 +1,109 @@ +import { lstat, realpath } from "node:fs/promises"; +import { dirname, isAbsolute, relative, resolve, sep } from "node:path"; + +import { MigrationPathSchema } from "@founderos/knowledge-schema"; + +import { SafePathError } from "../domain/safe-path.js"; + +function isInside(rootPath: string, candidatePath: string): boolean { + const relativePath = relative(rootPath, candidatePath); + return ( + relativePath === "" || + (!isAbsolute(relativePath) && relativePath !== ".." && !relativePath.startsWith(`..${sep}`)) + ); +} + +async function assertNoSymbolicLinkComponents( + rootPath: string, + logicalPath: string, + includeLeaf: boolean, +): Promise { + const segments = logicalPath.split("/"); + const limit = includeLeaf ? segments.length : segments.length - 1; + let currentPath = rootPath; + + for (const segment of segments.slice(0, limit)) { + currentPath = resolve(currentPath, segment); + const status = await lstat(currentPath); + if (status.isSymbolicLink()) { + throw new SafePathError(`Symbolic links are not allowed in migration paths: ${logicalPath}`); + } + } +} + +export async function resolvePhysicalRoot(rootPath: string): Promise { + const resolvedRoot = resolve(rootPath); + const status = await lstat(resolvedRoot); + + if (status.isSymbolicLink() || !status.isDirectory()) { + throw new SafePathError("Migration root must be a physical directory"); + } + + return realpath(resolvedRoot); +} + +export async function resolveSafeExistingFile( + physicalRoot: string, + logicalPath: string, +): Promise { + const validation = MigrationPathSchema.safeParse(logicalPath); + if (!validation.success) { + throw new SafePathError(`Unsafe migration path: ${logicalPath}`); + } + + const resolvedPath = resolve(physicalRoot, ...logicalPath.split("/")); + if (!isInside(physicalRoot, resolvedPath)) { + throw new SafePathError(`Migration path escapes the approved root: ${logicalPath}`); + } + + await assertNoSymbolicLinkComponents(physicalRoot, logicalPath, true); + const status = await lstat(resolvedPath); + if (!status.isFile() || status.isSymbolicLink()) { + throw new SafePathError(`Migration path must identify a physical file: ${logicalPath}`); + } + + const physicalPath = await realpath(resolvedPath); + if (!isInside(physicalRoot, physicalPath)) { + throw new SafePathError(`Migration path resolves outside the approved root: ${logicalPath}`); + } + + return physicalPath; +} + +export async function resolveSafeOutputPath( + physicalRoot: string, + logicalPath: string, +): Promise { + const validation = MigrationPathSchema.safeParse(logicalPath); + if (!validation.success) { + throw new SafePathError(`Unsafe migration output path: ${logicalPath}`); + } + + if (logicalPath !== "migration-report.json") { + throw new SafePathError("Migration output path must be migration-report.json"); + } + + const resolvedPath = resolve(physicalRoot, ...logicalPath.split("/")); + if (!isInside(physicalRoot, resolvedPath)) { + throw new SafePathError(`Migration output escapes the approved root: ${logicalPath}`); + } + + await assertNoSymbolicLinkComponents(physicalRoot, logicalPath, false); + const parentPath = await realpath(dirname(resolvedPath)); + if (!isInside(physicalRoot, parentPath)) { + throw new SafePathError(`Migration output resolves outside the approved root: ${logicalPath}`); + } + + try { + const status = await lstat(resolvedPath); + if (!status.isFile() || status.isSymbolicLink()) { + throw new SafePathError(`Migration output must be a physical file: ${logicalPath}`); + } + } catch (error: unknown) { + if (!(error instanceof Error && "code" in error && error.code === "ENOENT")) { + throw error; + } + } + + return resolvedPath; +} diff --git a/services/knowledge-engine/src/interfaces/migration-report.ts b/services/knowledge-engine/src/interfaces/migration-report.ts new file mode 100644 index 0000000..71cab94 --- /dev/null +++ b/services/knowledge-engine/src/interfaces/migration-report.ts @@ -0,0 +1,72 @@ +import type { + KnowledgeObject, + KnowledgeObjectType, + MigrationStatus, + ReviewStatus, +} from "@founderos/knowledge-schema"; + +export const KNOWLEDGE_MIGRATION_REPORT_VERSION = "1.0" as const; + +export type MigrationErrorCode = + | "root_path_error" + | "manifest_path_unsafe" + | "manifest_read_error" + | "manifest_parse_error" + | "manifest_validation_error" + | "source_path_unsafe" + | "source_missing" + | "source_read_error" + | "source_hash_mismatch" + | "migration_status_not_ready" + | "review_not_approved" + | "knowledge_validation_error"; + +export interface MigrationError { + code: MigrationErrorCode; + fieldPath?: string; + message: string; +} + +interface MigrationDocumentReportBase { + destinationPath: string; + expectedSourceHash: string; + id: string; + migrationStatus: MigrationStatus; + objectType: KnowledgeObjectType; + reviewStatus: ReviewStatus; + sourcePath: string; +} + +export interface AcceptedMigrationDocumentReport extends MigrationDocumentReportBase { + actualSourceHash: string; + byteLength: number; + object: KnowledgeObject; + status: "accepted"; +} + +export interface RejectedMigrationDocumentReport extends MigrationDocumentReportBase { + actualSourceHash?: string; + byteLength?: number; + errors: MigrationError[]; + status: "rejected"; +} + +export type MigrationDocumentReport = + AcceptedMigrationDocumentReport | RejectedMigrationDocumentReport; + +export interface KnowledgeMigrationSummary { + acceptedDocuments: number; + byObjectType: Record; + rejectedDocuments: number; + totalDocuments: number; +} + +export interface KnowledgeMigrationReport { + corpusId: string | null; + documents: MigrationDocumentReport[]; + errors: MigrationError[]; + manifestPath: string; + schemaVersion: typeof KNOWLEDGE_MIGRATION_REPORT_VERSION; + status: "accepted" | "rejected"; + summary: KnowledgeMigrationSummary; +} diff --git a/services/knowledge-engine/tests/migration.test.ts b/services/knowledge-engine/tests/migration.test.ts new file mode 100644 index 0000000..3b8cab6 --- /dev/null +++ b/services/knowledge-engine/tests/migration.test.ts @@ -0,0 +1,303 @@ +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { afterEach, describe, expect, it } from "vitest"; +import { stringify } from "yaml"; + +import { + executeKnowledgeMigration, + runMigrationCommand, + serializeKnowledgeMigrationReport, +} from "../src/index.js"; + +const REPOSITORY_ROOT = resolve(fileURLToPath(new URL("../../../", import.meta.url))); +const MANIFEST_PATH = "knowledge/migration-manifest.yaml"; +const PRIORITY_ONE_SOURCE_PATHS = [ + "docs/architecture/FounderOS_Data_Architecture_Specification_v1.0.md", + "docs/architecture/FounderOS_MCP_Architecture_Specification_v1.0.md", + "docs/architecture/FounderOS_Repository_Architecture_Specification_v1.0.md", + "docs/architecture/FounderOS_Security_and_Governance_Architecture_Specification_v1.0.md", + "docs/architecture/FounderOS_System_Architecture_Specification_v1.0.md", + "docs/governance/FounderOS_Constitution_v1.0.md", + "docs/governance/FounderOS_Decision_Framework_v1.0.md", + "docs/governance/FounderOS_Design_Principles_v1.0.md", +] as const; +const temporaryDirectories: string[] = []; + +function sha256(source: string | Buffer): string { + return createHash("sha256").update(source).digest("hex"); +} + +function manifestDocument(sourceHash: string) { + return { + id: "test-knowledge-v1", + objectType: "knowledge", + sourcePath: "docs/source.md", + destinationPath: "knowledge/research/test-knowledge-v1.md", + sourceHash, + migrationStatus: "ready", + reviewStatus: "approved", + metadata: { + title: "Test knowledge", + domain: "FounderOS", + createdAt: "2026-07-28", + updatedAt: "2026-07-28", + status: "active", + confidence: "high", + importance: "high", + validationStatus: "validated", + tags: ["test"], + relationships: [], + }, + }; +} + +function manifestWith(documents: unknown[]) { + return { + schemaVersion: "1.0", + corpusId: "test-corpus", + documents, + }; +} + +async function temporaryRoot(source = "# Canonical source\n"): Promise<{ + manifestPath: string; + rootPath: string; + source: string; +}> { + const rootPath = await mkdtemp(resolve(tmpdir(), "founderos-migration-")); + temporaryDirectories.push(rootPath); + await mkdir(resolve(rootPath, "docs")); + await writeFile(resolve(rootPath, "docs/source.md"), source, "utf8"); + return { manifestPath: "manifest.yaml", rootPath, source }; +} + +async function writeManifest(rootPath: string, manifest: unknown): Promise { + await writeFile(resolve(rootPath, "manifest.yaml"), stringify(manifest), "utf8"); +} + +afterEach(async () => { + await Promise.all( + temporaryDirectories + .splice(0) + .map((directory) => rm(directory, { force: true, recursive: true })), + ); +}); + +describe("FounderOS Priority 1 migration", () => { + it("migrates all eight canonical documents with provenance and no source mutation", async () => { + const before = await Promise.all( + PRIORITY_ONE_SOURCE_PATHS.map((path) => readFile(resolve(REPOSITORY_ROOT, path))), + ); + + const report = await executeKnowledgeMigration({ + manifestPath: MANIFEST_PATH, + rootPath: REPOSITORY_ROOT, + }); + + const after = await Promise.all( + PRIORITY_ONE_SOURCE_PATHS.map((path) => readFile(resolve(REPOSITORY_ROOT, path))), + ); + expect(report.status).toBe("accepted"); + expect(report.summary).toMatchObject({ + acceptedDocuments: 8, + byObjectType: { knowledge: 8 }, + rejectedDocuments: 0, + totalDocuments: 8, + }); + expect(report.documents.map((document) => document.sourcePath)).toEqual( + PRIORITY_ONE_SOURCE_PATHS, + ); + expect(after).toEqual(before); + + for (const document of report.documents) { + expect(document.status).toBe("accepted"); + if (document.status === "accepted") { + expect(document.actualSourceHash).toBe(document.expectedSourceHash); + expect(document.object.metadata).toMatchObject({ + id: document.id, + objectType: document.objectType, + source: { + originalCreator: "FounderOS", + sourceReference: document.sourcePath, + sourceType: "official_specification", + }, + }); + expect(document.object).toHaveProperty( + "content", + (await readFile(resolve(REPOSITORY_ROOT, document.sourcePath), "utf8")).trim(), + ); + } + } + }); + + it("produces byte-identical reports and a correct report artifact", async () => { + const first = await executeKnowledgeMigration({ + manifestPath: MANIFEST_PATH, + rootPath: REPOSITORY_ROOT, + }); + const second = await executeKnowledgeMigration({ + manifestPath: MANIFEST_PATH, + rootPath: REPOSITORY_ROOT, + }); + + expect(serializeKnowledgeMigrationReport(first)).toBe( + serializeKnowledgeMigrationReport(second), + ); + + const temporary = await temporaryRoot(); + await writeManifest( + temporary.rootPath, + manifestWith([manifestDocument(sha256(temporary.source))]), + ); + await runMigrationCommand({ + ...temporary, + outputPath: "migration-report.json", + }); + const artifact = await readFile(resolve(temporary.rootPath, "migration-report.json"), "utf8"); + const parsed = JSON.parse(artifact) as { schemaVersion: string; status: string }; + + expect(artifact.endsWith("\n")).toBe(true); + expect(parsed).toMatchObject({ schemaVersion: "1.0", status: "accepted" }); + }); +}); + +describe("migration manifest failures", () => { + it("rejects duplicate object IDs before reading sources", async () => { + const temporary = await temporaryRoot(); + const first = manifestDocument(sha256(temporary.source)); + await writeManifest( + temporary.rootPath, + manifestWith([first, { ...first, destinationPath: "knowledge/research/duplicate.md" }]), + ); + + const report = await executeKnowledgeMigration(temporary); + + expect(report).toMatchObject({ + documents: [], + status: "rejected", + summary: { totalDocuments: 0 }, + }); + expect(report.errors).toHaveLength(2); + expect(report.errors.every((error) => error.code === "manifest_validation_error")).toBe(true); + }); + + it("reports a missing canonical source document", async () => { + const temporary = await temporaryRoot(); + const document = { + ...manifestDocument("0".repeat(64)), + sourcePath: "docs/missing.md", + }; + await writeManifest(temporary.rootPath, manifestWith([document])); + + const report = await executeKnowledgeMigration(temporary); + + expect(report.documents[0]).toMatchObject({ + errors: [{ code: "source_missing" }], + status: "rejected", + }); + }); + + it("rejects invalid object types and metadata", async () => { + const temporary = await temporaryRoot(); + const base = manifestDocument(sha256(temporary.source)); + await writeManifest( + temporary.rootPath, + manifestWith([ + { + ...base, + objectType: "architecture", + metadata: { ...base.metadata, status: "validated" }, + }, + ]), + ); + + const report = await executeKnowledgeMigration(temporary); + + expect(report.status).toBe("rejected"); + expect(report.errors.map((error) => error.fieldPath)).toEqual( + expect.arrayContaining(["documents.0.objectType", "documents.0.metadata.status"]), + ); + }); + + it("rejects source hash mismatches with expected and actual evidence", async () => { + const temporary = await temporaryRoot(); + await writeManifest(temporary.rootPath, manifestWith([manifestDocument("0".repeat(64))])); + + const report = await executeKnowledgeMigration(temporary); + + expect(report.documents[0]).toMatchObject({ + actualSourceHash: sha256(temporary.source), + errors: [{ code: "source_hash_mismatch", fieldPath: "sourceHash" }], + expectedSourceHash: "0".repeat(64), + status: "rejected", + }); + }); + + it("rejects entries that are not ready and human-approved", async () => { + const temporary = await temporaryRoot(); + const document = { + ...manifestDocument(sha256(temporary.source)), + migrationStatus: "pending", + reviewStatus: "pending", + }; + await writeManifest(temporary.rootPath, manifestWith([document])); + + const report = await executeKnowledgeMigration(temporary); + + expect(report.documents[0]).toMatchObject({ + errors: [{ code: "migration_status_not_ready" }], + status: "rejected", + }); + }); +}); + +describe("migration path safety", () => { + it("rejects lexical traversal in source and output paths", async () => { + const temporary = await temporaryRoot(); + const document = { + ...manifestDocument(sha256(temporary.source)), + sourcePath: "../outside.md", + }; + await writeManifest(temporary.rootPath, manifestWith([document])); + + const report = await executeKnowledgeMigration(temporary); + + expect(report.errors).toEqual([ + expect.objectContaining({ + code: "manifest_validation_error", + fieldPath: "documents.0.sourcePath", + }), + ]); + await expect( + runMigrationCommand({ ...temporary, outputPath: "../migration-report.json" }), + ).rejects.toThrow("Unsafe migration output path"); + await expect(runMigrationCommand({ ...temporary, outputPath: "package.json" })).rejects.toThrow( + "Migration output path must be migration-report.json", + ); + }); + + it("rejects a source symlink that points outside the approved root", async () => { + const temporary = await temporaryRoot(); + const outsideRoot = await temporaryRoot("# Outside\n"); + await symlink( + resolve(outsideRoot.rootPath, "docs/source.md"), + resolve(temporary.rootPath, "docs/linked.md"), + ); + const document = { + ...manifestDocument(sha256(outsideRoot.source)), + sourcePath: "docs/linked.md", + }; + await writeManifest(temporary.rootPath, manifestWith([document])); + + const report = await executeKnowledgeMigration(temporary); + + expect(report.documents[0]).toMatchObject({ + errors: [{ code: "source_path_unsafe" }], + status: "rejected", + }); + }); +});