feat: add routing catalog and maintained eval program

.github/workflows/ci.yml

@@ -53,9 +53,10 @@ jobs:
   #
   # There is no root pyproject.toml or lockfile — this repo is a BUNDLE, not a
   # Python package, and only modules/* are installable. So the root suite runs
-  # in an ephemeral uv environment with its three actual dependencies. The
+  # in an ephemeral uv environment with test and real runtime dependencies. The
   # tests reach modules/hooks-routing via a sys.path insert of their own (see
-  # tests/test_default_resolution_unchanged.py), so nothing needs installing.
+  # tests/test_default_resolution_unchanged.py). Catalog lifecycle/resolver tests
+  # need real Core/Foundation, not a fixture stub or dependency-induced skip.
   # ---------------------------------------------------------------------------
   test-root:
     name: Tests — root (Python ${{ matrix.python-version }})
@@ -79,7 +80,8 @@ jobs:
         run: >
           uv run --no-project --python ${{ matrix.python-version }}
           --with pytest --with pytest-asyncio --with pyyaml --with click
-          python -m pytest tests -q --tb=short
+          --with git+https://github.com/microsoft/amplifier-foundation@main
+          python -c "import amplifier_core, amplifier_core.models, amplifier_foundation.spawn_utils, pytest; raise SystemExit(pytest.main(['tests', '-q', '--tb=short']))"
 
   # ---------------------------------------------------------------------------
   # Per-module test suites. Each modules/* package owns its own pyproject.toml
@@ -101,6 +103,8 @@ jobs:
   #    purpose. Two tests guard themselves with `pytest.importorskip`, which
   #    means a botched install SKIPS them silently instead of failing. The
   #    explicit import step below converts that silent skip into a red run.
+  #    Pre-import again INSIDE the pytest process: otherwise conftest can install
+  #    a Core stub even when the separate import step succeeded.
   # ---------------------------------------------------------------------------
   test-modules:
     name: Tests — modules/${{ matrix.module }}
@@ -135,7 +139,7 @@ jobs:
         run: >
           uv run --frozen --extra dev
           --with git+https://github.com/microsoft/amplifier-foundation
-          python -m pytest -q --tb=short
+          python -c "import amplifier_core, amplifier_core.models, amplifier_foundation.spawn_utils, pytest; raise SystemExit(pytest.main(['-q', '--tb=short']))"
 
       - name: Run catalog refresh runtime seam tests with real dependencies
         working-directory: modules/${{ matrix.module }}
@@ -144,6 +148,41 @@ jobs:
           --with git+https://github.com/microsoft/amplifier-foundation
           python -c "import amplifier_core, amplifier_foundation.spawn_utils, pytest; raise SystemExit(pytest.main(['../../tests/test_catalog_refresh.py', '-q', '--tb=short']))"
 
+  # Maintained routing-specific offline contracts, not a paid evaluation runner.
+  # The second checkout is public pinned task data only: never execute its scripts
+  # or install its evaluator runtime. No secrets or result-artifact uploads.
+  routing-evidence:
+    name: Routing evidence — Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          repository: microsoft/amplifier-bundle-evaluation
+          ref: 7c3646796eea2b042bbe6ffb2de3906d31daa381
+          path: .eval-deps/amplifier-bundle-evaluation
+          persist-credentials: false
+          fetch-depth: 1
+      - uses: astral-sh/setup-uv@ae62891fec2bb8e7d6c99fc78c9fec3a63790f8d # v10.0.0
+      - name: Run offline contract tests
+        env:
+          PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1"
+          PYTHONDONTWRITEBYTECODE: "1"
+          ROUTING_EVAL_BENCHMARK_ROOT: ${{ github.workspace }}/.eval-deps/amplifier-bundle-evaluation/amplifier-benchmark/tasks
+        run: >-
+          uv run --no-project --python ${{ matrix.python-version }}
+          --with pytest --with click --with pyyaml
+          python -m pytest -q -p no:cacheprovider --import-mode=importlib evals
+      - name: Lint offline routing evidence
+        run: uvx ruff@0.15.11 check evals
+      - name: Check offline routing evidence formatting
+        # Retain the original library bytes (88 columns), not whole-repo formatting.
+        run: uvx ruff@0.15.11 format --check --config line-length=88 evals
+
   # ---------------------------------------------------------------------------
   # Bundle structure — a cheap YAML parse of bundle.md's frontmatter, every
   # behaviors/*.yaml and every routing/*.yaml. No network, no keys, ~1 second.

.gitignore

@@ -6,3 +6,7 @@ __pycache__/
 dist/
 build/
 .pytest_cache/
+# Pinned public evaluation assets are a dev/CI cache, not routing source.
+/.eval-deps/
+# Safeguard only: raw results belong in private ignored storage outside repos.
+/evals/results/

AGENTS.md

@@ -0,0 +1,43 @@
+# Routing bundle: agent handoff
+
+## Read before work
+
+Read `README.md`, `docs/HANDOFF.md`, `docs/VISION.md`, and the documents relevant to your change:
+- Catalog, identity, scope or host integration: `contracts/routing-catalog.v1.md`, `docs/CATALOG-SPEC.md`, `docs/PROVIDER_AVAILABILITY.md`.
+- Model preferences, evaluation or evidence: `docs/EVALUATION-PROGRAM.md`, `docs/MATRIX_CURATOR_GUIDE.md`.
+- Offline routing evaluation tooling: `evals/AGENTS.md` and `evals/README.md`.
+- Tests and release gates: `.github/workflows/ci.yml`, `ruff.toml`, module `pyproject.toml` and module tests.
+
+The new vision/catalog contract are **DRAFT**. They describe proposed direction, not delivered behavior, frozen promises or permission to implement. Handoff distinguishes current source facts, proposed changes and decisions still needed. Re-read local conventions at design/code/verification/publication transitions.
+
+## Ownership and invariants
+
+This bundle owns matrix identities, aliases, role definitions, preference policy and resolver semantics. Consumers own presentation, configuration persistence and credential authority. Do not import a consumer application or evaluation runner into the runtime hook. Portable dispatch enforcement is a separately qualified integration dependency, not a capability this routing hook already supplies.
+
+Routing-specific evaluation configurations, scenarios, graders, reuse rules and promotion policy are maintained in `evals/` and `docs/EVALUATION-PROGRAM.md`. Evaluation assets/library bricks are optional development dependencies, never runtime hook imports or production dependencies. The offline tools require an explicit local benchmark root; the separate CI job acquires pinned public assets without executing upstream scripts. Keep that dependency isolated from routing policy and ordinary root/module tests.
+
+Preserve custom-file precedence, explicit intent, caller inheritance, exact mounted-instance provenance and failure behavior. The current `openai` matrix is API-first across API and ChatGPT; it is **not API-only**. Do not silently change saved IDs, widen account scope, invent fallback IDs, or label wildcard/template discovery verified. Catalog membership and entitlement are different evidence.
+
+The bounded catalog library is documented in `docs/catalog-api.md`. It reuses effective resolver semantics with snapshots and independent initial state; never pass live escalation state, reporting sinks or mutable caches to assessment. Catalog calls must not reach runtime logger handlers: preserve runtime diagnostics by explicit reporting injection, not global logger disabling. Explicit-preference assessment and next-dispatch prediction remain unsupported; broader specification examples are not executable interfaces.
+
+## Verification
+
+The root and routing-module suites both matter. Use the commands and supported Python matrix from `.github/workflows/ci.yml`; run `python -m pytest`, not an accidentally isolated `pytest` entry point. Module checks must actually import Foundation/Core where required; a dependency-induced skip is not a pass. CI pins Ruff at `0.15.11`; do not add unrelated whole-repo formatting. Run `.github/scripts/check_bundle_structure.py` for YAML/frontmatter changes.
+
+Runtime/provider/spawn changes also need an isolated full-stack smoke with real installed dependencies. Record environment readiness before the check, exact revisions, selected test counts, actual results and skips. Unit or catalog checks do not establish live model quality, account containment or deployed adoption. Keep regressions present when reverting production to prove a real red/green pair.
+
+Root CI now needs actual Core/Foundation because catalog tests exercise lifecycle and capability paths. Pre-import them in the **same pytest process** before module fixtures can install a stub; a separate import command is insufficient. Canonical/alias collapse also needs the entire winning-source group to match: a custom canonical profile must not suppress a still-valid bundled alias strategy.
+
+No inference or recurring paid benchmarks in ordinary offline CI. Live runs need an approved token/time/spend envelope, private ignored output storage, lifecycle accounting and safe credentials; parsed-but-inert preset knobs are not budget controls.
+
+Run the independent `evals/` suite with `ROUTING_EVAL_BENCHMARK_ROOT` explicitly set as documented there. Missing assets fail, never skip or fall back to ambient sibling checkouts. Preserve historical sample source/task/grader locks during relocation; they do not attest the current routing revision. `/.eval-deps/` is an ignored asset cache and `/evals/results/` is only an accidental-output safeguard, not an approved raw-result location.
+
+## Publication and privacy
+
+Commit only newly authored synthetic fixtures, approved public-source metadata and schema-allowlisted summaries. Never commit raw captures, secret/config-derived hashes, personal identities/paths/endpoints, account bindings, private session maps, holdout answers or private project links. Being visible in a local session is not publication permission. Public attribution must cite an already-public source and disclose no associated private identity.
+
+New content classes require a fresh-context stranger/leak review before push; scan staged content and any uploaded artifacts. Do not copy a private governance document into this public bundle. Keep raw acquisition, subject execution and grading isolated from personal home/browser/credential stores. Do not delete source records merely to produce sanitized artifacts.
+
+## Changes to direction
+
+Drafts may be revised as drafts. A steward-locked document changes through an evidence-backed candidate proposal, never silent edits. Do not mark a document frozen or create a passing ledger from prose. The lock bar needs a discriminating good/bad kit, a passing real implementation, an end-to-end example and the steward's decision. Review approval is not ratification.