Add ORT GenAI end-to-end CI

.agents/skills/quality-checklist/SKILL.md

@@ -165,9 +165,13 @@ python examples/<model>_text_generation.py --compare-hf --dtype bf16
 
 Downstream evidence:
 
-- [ ] If ORT GenAI load/generation is run, record the exact runtime version and
-      result. Failures document limitations but never gate Mobius export based
-      on the runtime registry, topology support, or cache executor capability.
+- [ ] The network-free generic decoder test passes with the pinned latest stable
+      `onnxruntime-genai==0.15.2`.
+- [ ] Every runtime-supported real route has an `ort_genai` YAML marker with an
+      immutable evidence ID, exact tokenizer provenance, bounded download size,
+      CPU provider claim, and explicit released-version capabilities.
+- [ ] Real generation asserts the full generated length before token equality
+      and runs with an isolated Hub/Xet cache that is deleted after the test.
 - [ ] If ORT GenAI validation is run for a model with dual head dimensions,
       determine whether that runtime requires
       `search.past_present_share_buffer=false` for its uniform KV-cache
@@ -183,7 +187,9 @@ Waiver needed if any of the steps are not possible.
 Run the ORT GenAI integration test:
 
 ```bash
-python -m pytest tests/ort_genai_test.py -m integration_slow -k "<model>" -sv
+python -m pytest tests/ort_genai_e2e_test.py -m ort_genai_fast -v
+python -m pytest tests/gguf_small_model_runtime_integration_test.py \
+  -m ort_genai_real -v
 ```
 
 ### 10. Foundry Local package check

.github/workflows/main.yml

@@ -218,6 +218,17 @@ jobs:
       HF_TOKEN: ${{ secrets.HF_TOKEN }}
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
+  ort-genai-e2e:
+    name: ORT GenAI E2E
+    needs: [detect-affected]
+    if: >-
+      always() && !cancelled() &&
+      (needs.detect-affected.result == 'failure' ||
+       needs.detect-affected.outputs.has_affected == 'true')
+    uses: ./.github/workflows/ort_genai_e2e.yml
+    with:
+      run_real: false
+
   test:
     name: test
     strategy:

.github/workflows/ort_genai_e2e.yml

@@ -0,0 +1,93 @@
+name: ORT GenAI End-to-End
+
+on:
+  schedule:
+    - cron: "0 7 * * 2"
+  workflow_dispatch:
+  workflow_call:
+    inputs:
+      run_real:
+        description: Run the pinned real-artifact lane.
+        required: false
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ort-genai-e2e-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  fast:
+    name: Fast CPU / OGA 0.15.2
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+      - name: Install pinned runtime and test dependencies
+        run: |
+          python -m pip install --disable-pip-version-check \
+            "onnxruntime-genai==0.15.2" \
+            "pytest==8.4.2" \
+            "pytest-timeout==2.4.0"
+          python -m pip install --disable-pip-version-check -e .
+      - name: Run network-free ORT GenAI E2E
+        env:
+          HF_HUB_OFFLINE: "1"
+          TRANSFORMERS_OFFLINE: "1"
+          MOBIUS_EXPECTED_ORT_GENAI_VERSION: "0.15.2"
+        run: |
+          python -m pytest tests/ort_genai_e2e_test.py \
+            -m ort_genai_fast -v --timeout=120 --junitxml=junit-ort-genai-fast.xml
+      - name: Upload fast-lane logs
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: ort-genai-fast-0.15.2
+          path: junit-ort-genai-fast.xml
+          retention-days: 14
+
+  real-smollm:
+    name: Pinned SmolLM F16 CPU / OGA 0.15.2
+    if: inputs.run_real || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    timeout-minutes: 35
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+      - name: Install pinned runtime and test dependencies
+        run: |
+          python -m pip install --disable-pip-version-check \
+            "onnxruntime-genai==0.15.2" \
+            "pytest==8.4.2" \
+            "pytest-timeout==2.4.0" \
+            "gguf==0.17.1"
+          python -m pip install --disable-pip-version-check -e ".[transformers]"
+      - name: Run pinned real-artifact generation
+        env:
+          MOBIUS_EXPECTED_ORT_GENAI_VERSION: "0.15.2"
+        run: |
+          python -m pytest tests/gguf_small_model_runtime_integration_test.py \
+            -m ort_genai_real -v --timeout=1500 --junitxml=junit-ort-genai-real.xml
+      - name: Upload real-lane logs
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: ort-genai-real-smollm-0.15.2
+          path: junit-ort-genai-real.xml
+          retention-days: 30

CONTRIBUTING.md

@@ -201,7 +201,11 @@ the PR description).  The full checklist with explanations lives in the
 
 **Runtime and deployment**
 - [ ] CLI build: `mobius build --model <hf-model-id> /tmp/out` succeeds
-- [ ] ORT GenAI: model loads and generates coherent output; test in `tests/ort_genai_test.py`
+- [ ] ORT GenAI fast lane passes on the pinned latest stable runtime:
+  `pytest tests/ort_genai_e2e_test.py -m ort_genai_fast -v`
+- [ ] Runtime-supported real routes declare `ort_genai` evidence in their
+  `testdata/cases/` YAML and pass:
+  `pytest tests/gguf_small_model_runtime_integration_test.py -m ort_genai_real -v`
 - [ ] Foundry Local: exported package loads and responds to a short prompt
 - [ ] Olive quantization: INT4/INT8 quantization runs to completion; quantized model produces coherent output
 

docs/api/build_from_gguf.md

@@ -421,7 +421,7 @@ before graph construction or durable output.
 | `lfm2moe` | — | model=`lfm2_moe`; tensor=`lfm2`+`lfm2_moe_extras` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=rejected | Config extraction, exact pinned tensor-name closure, GGUF value transforms, and synthetic recurrent-state execution are covered, but no representative real-weight GGUF has yet passed independent full-logit parity and deterministic multi-token stateful ORT generation. Runtime packaging remains deferred until that evidence exists. The mobius graph uses floating Linear modules for this architecture, so no MatMulNBits or BlockQuantizedMatMul target can consume preserved GGUF projection weights. Use keep_quantized=False for explicit float import. |
 | `llada` | — | model=`llada`; tensor=`llama` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=supported | Config extraction, suffix-exact tensor closure, masked-diffusion task dispatch, and synthetic full-sequence execution are covered, but no pinned real GGUF has passed independent Hugging Face/llama.cpp masked-step logit parity and deterministic multi-step generation parity. Runtime packaging remains deferred until both exist. |
 | `llada-moe` | — | model=`llada`; module=`llada_moe`; tensor=`llama`+`diffusion_fused_qkv`+`moe_qk_norm_extras`+`moe_extras` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=supported | Config extraction, suffix-exact tensor closure, masked-diffusion task dispatch, and synthetic full-sequence execution are covered, but no pinned real GGUF has passed independent Hugging Face/llama.cpp masked-step logit parity and deterministic multi-step generation parity. Runtime packaging remains deferred until both exist. |
-| `llama` | `mistral` | model=`llama`; tensor=`llama` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=supported; quantized_import=supported | Runtime support is restricted to exact structured evidence matches. Currently that is only neopolita/smollm-135m-gguf F16 at the pinned artifact, CPU import route, ONNX Runtime 1.29.0, and HuggingFaceTB/SmolLM-135M tokenizer revision. |
+| `llama` | `mistral` | model=`llama`; tensor=`llama` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=supported; quantized_import=supported | Runtime support is restricted to exact structured evidence matches. Currently that is only neopolita/smollm-135m-gguf F16 at the pinned artifact, CPU import route, evidenced ONNX Runtime/ORT GenAI versions, and the pinned HuggingFaceTB/SmolLM-135M tokenizer revision. |
 | `llama-embed` | — | none (fails before config extraction) | audited-direct-loader-conditional-union | config=deferred; tensor_map=deferred; graph=deferred; runtime=deferred; quantized_import=supported | llama-embed is a canonical embedding architecture that inherits Llama's conditional tensor loader but exposes the embedding graph rather than causal logits. Mobius has no GGUF embedding task/package contract for this ID, so it must not alias ordinary llama. |
 | `llama4` | — | none (fails before config extraction) | exact-direct-loader-conditional-union | config=deferred; tensor_map=deferred; graph=deferred; runtime=deferred; quantized_import=supported | Llama4 GGUF is the text member of a paired multimodal package and may contain routed experts and architecture-specific cross-modal layer scheduling. The llama4 clip vision tower, token mixing, position IDs, and package ABI remain deferred; text-backbone similarity is not evidence that the complete GGUF tensor closure is owned. |
 | `maincoder` | — | none (fails before config extraction) | audited-direct-loader-conditional-union | config=deferred; tensor_map=deferred; graph=deferred; runtime=deferred; quantized_import=supported | Maincoder applies Q/K RMSNorm after RoPE and uses an exact tied-output QK-normalized SwiGLU closure. Existing generic QK-normalized graphs use different ordering, so a family alias would change attention. |

docs/design/multi-tier-testing-strategy.md

@@ -862,12 +862,34 @@ The test must verify:
 3. Image preprocessing config (resize, normalize, crop) matches HF
 4. Tokenizer and processor files are present and well-formed
 
-### Proposed test tiers for ORT GenAI
+### ORT GenAI runtime tiers
 
-| Level | What it tests                        | Cost     | CI cadence    |
-|-------|--------------------------------------|----------|---------------|
-| L1+   | Config generation (no weights)       | Seconds  | Every PR      |
-| E2E   | Full pipeline: export + load + gen   | Minutes  | Weekly / GPU   |
+| Lane | What it tests | Runtime | CI cadence |
+|------|---------------|---------|------------|
+| Fast | Generated generic `decoder` package, tokenizer load, model creation, prefill, four cache-threaded decode steps, deterministic tokens, save/reload, and malformed config rejection | CPU; latest stable `onnxruntime-genai==0.15.2` | Relevant PRs, main, scheduled/manual |
+| Real | YAML-enrolled immutable artifacts with exact tokenizer provenance, byte budget, deterministic full-length generation, and disposable per-test Hub/Xet caches | CPU; currently `onnxruntime-genai==0.15.2` | Weekly and manual |
+
+Run the lanes locally:
+
+```bash
+python -m pytest tests/ort_genai_e2e_test.py -m ort_genai_fast -v
+python -m pytest tests/gguf_small_model_runtime_integration_test.py \
+  -m ort_genai_real -v
+```
+
+The `ort_genai` object in each golden-case YAML is the enrollment record.
+Schema and coverage tests fail when a runtime-supported GGUF route lacks a
+matching evidence ID. Released lanes use the architecture-neutral
+`model.type=decoder`; main-only `state_groups` metadata is explicitly excluded.
+Changes to graph construction, task wiring, ORT GenAI config generation,
+tokenizer packaging, and GGUF qtype/runtime helpers conservatively select the
+fast E2E lane through `detect_affected_models.py`.
+
+CUDA is not a required PR claim. The existing self-hosted GPU lane uses a
+CUDA-12-specific pre-release ORT feed, so adding it to this stable-wheel
+lane would conflate runtime and EP coverage. CUDA runtime E2E remains suitable
+for scheduled/manual validation once that runner can install an equivalently
+pinned released stack.
 
 ---
 

pyproject.toml

@@ -78,6 +78,8 @@ markers = [
     "arch_validation: L2 architecture validation tests that download real HF configs (no weights) and build full-size ONNX graphs (deselect with '-m \"not arch_validation\"')",
     "golden: L4 checkpoint-verified golden comparison tests (deselect with '-m \"not golden\"')",
     "generation: L5 generation end-to-end golden tests (deselect with '-m \"not generation\"')",
+    "ort_genai_fast: network-free end-to-end tests against released onnxruntime-genai wheels",
+    "ort_genai_real: pinned real-artifact onnxruntime-genai generation tests",
 ]
 
 [tool.mypy]

scripts/detect_affected_models.py

@@ -35,15 +35,33 @@
 _SRC_ROOT = _PROJECT_ROOT / "src" / "mobius"
 
 # ----------------------------------------------------------------
-# Shared infrastructure paths — any change triggers run_all.
-# Currently empty: core infra files (e.g. _configs.py, _registry.py)
-# are NOT under _TRACEABLE_PREFIXES and therefore classify as "other",
-# triggering no model tests. Add entries here to re-enable run_all for
-# specific paths when broader regression coverage is needed.
+# Shared graph/runtime infrastructure paths — any change triggers run_all.
+# This is intentionally limited to surfaces that can alter exported model
+# graphs, runtime metadata, tokenizer assets, or GGUF qtype handling.
 # ----------------------------------------------------------------
-_SHARED_INFRA_PATTERNS: tuple[str, ...] = ()
+_SHARED_INFRA_PATTERNS: tuple[str, ...] = (
+    "src/mobius/integrations/gguf/_runtime_evidence.py",
+    "src/mobius/integrations/gguf/_runtime_package.py",
+    "src/mobius/integrations/gguf/_tokenizer.py",
+    "src/mobius/integrations/gguf/_builder.py",
+    "src/mobius/integrations/gguf/_quant_registry.py",
+    "src/mobius/integrations/gguf/_repacker.py",
+    "src/mobius/integrations/gguf/_reader.py",
+    "src/mobius/_builder.py",
+    "src/mobius/_model_package.py",
+    "src/mobius/_optimizations.py",
+    "src/mobius/_weight_loading.py",
+    "tests/ort_genai_e2e_test.py",
+    "tests/gguf_small_model_runtime_integration_test.py",
+    "testdata/cases/schema.json",
+    ".github/workflows/ort_genai_e2e.yml",
+    "pyproject.toml",
+)
 
-_SHARED_INFRA_PREFIXES: tuple[str, ...] = ()
+_SHARED_INFRA_PREFIXES: tuple[str, ...] = (
+    "src/mobius/integrations/ort_genai/",
+    "src/mobius/tasks/",
+)
 
 # Traceable infrastructure: component files that are analyzed via the
 # import graph to find which models they actually affect, rather than
@@ -62,6 +80,11 @@ def classify_file(path: str) -> str:
     """
     normalized = path.replace("\\", "/")
 
+    if normalized in _SHARED_INFRA_PATTERNS:
+        return "shared_infra"
+    if any(normalized.startswith(prefix) for prefix in _SHARED_INFRA_PREFIXES):
+        return "shared_infra"
+
     if not normalized.startswith("src/mobius/"):
         # Test infrastructure files that affect all models
         if normalized == "tests/conftest.py":

scripts/detect_affected_models_test.py

@@ -49,7 +49,7 @@ def test_component_file(self):
         assert classify_file("src/mobius/components/_attention.py") == "traceable"
 
     def test_task_file(self):
-        assert classify_file("src/mobius/tasks/_causal_lm.py") == "traceable"
+        assert classify_file("src/mobius/tasks/_causal_lm.py") == "shared_infra"
 
     def test_configs_file(self):
         assert classify_file("src/mobius/_configs.py") == "other"
@@ -58,7 +58,7 @@ def test_registry_file(self):
         assert classify_file("src/mobius/_registry.py") == "other"
 
     def test_builder_file(self):
-        assert classify_file("src/mobius/_builder.py") == "other"
+        assert classify_file("src/mobius/_builder.py") == "shared_infra"
 
     def test_exporter_file(self):
         assert classify_file("src/mobius/_exporter.py") == "other"
@@ -85,7 +85,15 @@ def test_readme(self):
         assert classify_file("README.md") == "other"
 
     def test_pyproject(self):
-        assert classify_file("pyproject.toml") == "other"
+        assert classify_file("pyproject.toml") == "shared_infra"
+
+    def test_ort_genai_integration_file(self):
+        assert (
+            classify_file("src/mobius/integrations/ort_genai/ep_config.py") == "shared_infra"
+        )
+
+    def test_ort_genai_workflow(self):
+        assert classify_file(".github/workflows/ort_genai_e2e.yml") == "shared_infra"
 
     def test_windows_paths(self):
         assert classify_file("src\\mobius\\models\\falcon.py") == "model"
@@ -224,16 +232,19 @@ def test_component_change_traces_affected_models(self):
         # _attention.py is imported by many models — should find affected types
         assert len(result["affected"]) > 0
 
-    def test_task_change_does_not_trigger_run_all(self):
-        """Task files are traceable but produce an empty affected set.
-
-        No model imports ``mobius.tasks`` directly (tasks are looked up at
-        runtime by string keys), so tracing through the import graph finds
-        no dependents. Documented limitation — see PR description.
-        """
+    def test_task_change_triggers_runtime_matrix(self):
         result = detect_affected_models(["src/mobius/tasks/_causal_lm.py"])
-        assert result["run_all"] is False
-        assert result["affected"] == []
+        assert result == {"affected": [], "run_all": True}
+
+    def test_ort_genai_change_triggers_runtime_matrix(self):
+        result = detect_affected_models(
+            ["src/mobius/integrations/ort_genai/_execution_providers.py"]
+        )
+        assert result == {"affected": [], "run_all": True}
+
+    def test_dependency_change_triggers_runtime_matrix(self):
+        result = detect_affected_models(["pyproject.toml"])
+        assert result == {"affected": [], "run_all": True}
 
     def test_configs_change_no_run_all(self):
         """_configs.py no longer triggers run_all (shared_infra disabled)."""
@@ -253,7 +264,7 @@ def test_test_configs_with_model_uses_model_scope(self):
             ]
         )
         assert result["run_all"] is False
-        assert result["affected"] == ["lfm2"]
+        assert result["affected"] == ["lfm2", "lfm2_moe", "lfm2_vl"]
 
     def test_test_configs_with_unmapped_task_still_runs_all(self):
         result = detect_affected_models(
@@ -348,18 +359,23 @@ def test_component_common_affects_many_models(self):
         # _common.py defines Linear, Embedding, LayerNorm — used everywhere
         assert len(result["affected"]) > 10
 
-    def test_former_shared_infra_no_run_all(self):
-        """Former shared_infra files no longer trigger run_all."""
+    def test_runtime_shared_infra_runs_all(self):
         for path in [
-            "src/mobius/_configs.py",
-            "src/mobius/_registry.py",
+            "src/mobius/_model_package.py",
+            "src/mobius/integrations/ort_genai/auto_export.py",
+            "src/mobius/integrations/ort_genai/genai_config.py",
+            "src/mobius/integrations/gguf/_runtime_package.py",
+            "src/mobius/integrations/gguf/_tokenizer.py",
+            "src/mobius/integrations/gguf/_builder.py",
+            "src/mobius/integrations/gguf/_quant_registry.py",
+            "src/mobius/integrations/gguf/_repacker.py",
             "src/mobius/_builder.py",
+            "src/mobius/_optimizations.py",
             "src/mobius/_weight_loading.py",
-            "src/mobius/_model_package.py",
-            "src/mobius/models/__init__.py",
+            "testdata/cases/schema.json",
         ]:
             result = detect_affected_models([path])
-            assert result["run_all"] is False, f"{path} should NOT trigger run_all"
+            assert result["run_all"] is True, f"{path} should trigger run_all"
 
     def test_traceable_and_model_combined(self):
         """A component + model file change returns union of affected types."""