diff --git a/.agents/skills/quality-checklist/SKILL.md b/.agents/skills/quality-checklist/SKILL.md index f539eb275..27a8d9675 100644 --- a/.agents/skills/quality-checklist/SKILL.md +++ b/.agents/skills/quality-checklist/SKILL.md @@ -165,9 +165,13 @@ python examples/_text_generation.py --compare-hf --dtype bf16 Downstream evidence: -- [ ] If ORT GenAI load/generation is run, record the exact runtime version and - result. Failures document limitations but never gate Mobius export based - on the runtime registry, topology support, or cache executor capability. +- [ ] The network-free generic decoder test passes with the pinned latest stable + `onnxruntime-genai==0.15.2`. +- [ ] Every runtime-supported real route has an `ort_genai` YAML marker with an + immutable evidence ID, exact tokenizer provenance, bounded download size, + CPU provider claim, and explicit released-version capabilities. +- [ ] Real generation asserts the full generated length before token equality + and runs with an isolated Hub/Xet cache that is deleted after the test. - [ ] If ORT GenAI validation is run for a model with dual head dimensions, determine whether that runtime requires `search.past_present_share_buffer=false` for its uniform KV-cache @@ -183,7 +187,9 @@ Waiver needed if any of the steps are not possible. Run the ORT GenAI integration test: ```bash -python -m pytest tests/ort_genai_test.py -m integration_slow -k "" -sv +python -m pytest tests/ort_genai_e2e_test.py -m ort_genai_fast -v +python -m pytest tests/gguf_small_model_runtime_integration_test.py \ + -m ort_genai_real -v ``` ### 10. Foundry Local package check diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d58f1860f..2fb0a91b6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -218,6 +218,17 @@ jobs: HF_TOKEN: ${{ secrets.HF_TOKEN }} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + ort-genai-e2e: + name: ORT GenAI E2E + needs: [detect-affected] + if: >- + always() && !cancelled() && + (needs.detect-affected.result == 'failure' || + needs.detect-affected.outputs.has_affected == 'true') + uses: ./.github/workflows/ort_genai_e2e.yml + with: + run_real: false + test: name: test strategy: diff --git a/.github/workflows/ort_genai_e2e.yml b/.github/workflows/ort_genai_e2e.yml new file mode 100644 index 000000000..b5af80a7a --- /dev/null +++ b/.github/workflows/ort_genai_e2e.yml @@ -0,0 +1,93 @@ +name: ORT GenAI End-to-End + +on: + schedule: + - cron: "0 7 * * 2" + workflow_dispatch: + workflow_call: + inputs: + run_real: + description: Run the pinned real-artifact lane. + required: false + type: boolean + default: false + +permissions: + contents: read + +concurrency: + group: ort-genai-e2e-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + fast: + name: Fast CPU / OGA 0.15.2 + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + - name: Install pinned runtime and test dependencies + run: | + python -m pip install --disable-pip-version-check \ + "onnxruntime-genai==0.15.2" \ + "pytest==8.4.2" \ + "pytest-timeout==2.4.0" + python -m pip install --disable-pip-version-check -e . + - name: Run network-free ORT GenAI E2E + env: + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + MOBIUS_EXPECTED_ORT_GENAI_VERSION: "0.15.2" + run: | + python -m pytest tests/ort_genai_e2e_test.py \ + -m ort_genai_fast -v --timeout=120 --junitxml=junit-ort-genai-fast.xml + - name: Upload fast-lane logs + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ort-genai-fast-0.15.2 + path: junit-ort-genai-fast.xml + retention-days: 14 + + real-smollm: + name: Pinned SmolLM F16 CPU / OGA 0.15.2 + if: inputs.run_real || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 35 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + - name: Install pinned runtime and test dependencies + run: | + python -m pip install --disable-pip-version-check \ + "onnxruntime-genai==0.15.2" \ + "pytest==8.4.2" \ + "pytest-timeout==2.4.0" \ + "gguf==0.17.1" + python -m pip install --disable-pip-version-check -e ".[transformers]" + - name: Run pinned real-artifact generation + env: + MOBIUS_EXPECTED_ORT_GENAI_VERSION: "0.15.2" + run: | + python -m pytest tests/gguf_small_model_runtime_integration_test.py \ + -m ort_genai_real -v --timeout=1500 --junitxml=junit-ort-genai-real.xml + - name: Upload real-lane logs + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ort-genai-real-smollm-0.15.2 + path: junit-ort-genai-real.xml + retention-days: 30 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d3f456ce0..91ec96e9b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -201,7 +201,11 @@ the PR description). The full checklist with explanations lives in the **Runtime and deployment** - [ ] CLI build: `mobius build --model /tmp/out` succeeds -- [ ] ORT GenAI: model loads and generates coherent output; test in `tests/ort_genai_test.py` +- [ ] ORT GenAI fast lane passes on the pinned latest stable runtime: + `pytest tests/ort_genai_e2e_test.py -m ort_genai_fast -v` +- [ ] Runtime-supported real routes declare `ort_genai` evidence in their + `testdata/cases/` YAML and pass: + `pytest tests/gguf_small_model_runtime_integration_test.py -m ort_genai_real -v` - [ ] Foundry Local: exported package loads and responds to a short prompt - [ ] Olive quantization: INT4/INT8 quantization runs to completion; quantized model produces coherent output diff --git a/docs/api/build_from_gguf.md b/docs/api/build_from_gguf.md index bdd4f79a7..e345b89f5 100644 --- a/docs/api/build_from_gguf.md +++ b/docs/api/build_from_gguf.md @@ -421,7 +421,7 @@ before graph construction or durable output. | `lfm2moe` | — | model=`lfm2_moe`; tensor=`lfm2`+`lfm2_moe_extras` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=rejected | Config extraction, exact pinned tensor-name closure, GGUF value transforms, and synthetic recurrent-state execution are covered, but no representative real-weight GGUF has yet passed independent full-logit parity and deterministic multi-token stateful ORT generation. Runtime packaging remains deferred until that evidence exists. The mobius graph uses floating Linear modules for this architecture, so no MatMulNBits or BlockQuantizedMatMul target can consume preserved GGUF projection weights. Use keep_quantized=False for explicit float import. | | `llada` | — | model=`llada`; tensor=`llama` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=supported | Config extraction, suffix-exact tensor closure, masked-diffusion task dispatch, and synthetic full-sequence execution are covered, but no pinned real GGUF has passed independent Hugging Face/llama.cpp masked-step logit parity and deterministic multi-step generation parity. Runtime packaging remains deferred until both exist. | | `llada-moe` | — | model=`llada`; module=`llada_moe`; tensor=`llama`+`diffusion_fused_qkv`+`moe_qk_norm_extras`+`moe_extras` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=supported | Config extraction, suffix-exact tensor closure, masked-diffusion task dispatch, and synthetic full-sequence execution are covered, but no pinned real GGUF has passed independent Hugging Face/llama.cpp masked-step logit parity and deterministic multi-step generation parity. Runtime packaging remains deferred until both exist. | -| `llama` | `mistral` | model=`llama`; tensor=`llama` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=supported; quantized_import=supported | Runtime support is restricted to exact structured evidence matches. Currently that is only neopolita/smollm-135m-gguf F16 at the pinned artifact, CPU import route, ONNX Runtime 1.29.0, and HuggingFaceTB/SmolLM-135M tokenizer revision. | +| `llama` | `mistral` | model=`llama`; tensor=`llama` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=supported; quantized_import=supported | Runtime support is restricted to exact structured evidence matches. Currently that is only neopolita/smollm-135m-gguf F16 at the pinned artifact, CPU import route, evidenced ONNX Runtime/ORT GenAI versions, and the pinned HuggingFaceTB/SmolLM-135M tokenizer revision. | | `llama-embed` | — | none (fails before config extraction) | audited-direct-loader-conditional-union | config=deferred; tensor_map=deferred; graph=deferred; runtime=deferred; quantized_import=supported | llama-embed is a canonical embedding architecture that inherits Llama's conditional tensor loader but exposes the embedding graph rather than causal logits. Mobius has no GGUF embedding task/package contract for this ID, so it must not alias ordinary llama. | | `llama4` | — | none (fails before config extraction) | exact-direct-loader-conditional-union | config=deferred; tensor_map=deferred; graph=deferred; runtime=deferred; quantized_import=supported | Llama4 GGUF is the text member of a paired multimodal package and may contain routed experts and architecture-specific cross-modal layer scheduling. The llama4 clip vision tower, token mixing, position IDs, and package ABI remain deferred; text-backbone similarity is not evidence that the complete GGUF tensor closure is owned. | | `maincoder` | — | none (fails before config extraction) | audited-direct-loader-conditional-union | config=deferred; tensor_map=deferred; graph=deferred; runtime=deferred; quantized_import=supported | Maincoder applies Q/K RMSNorm after RoPE and uses an exact tied-output QK-normalized SwiGLU closure. Existing generic QK-normalized graphs use different ordering, so a family alias would change attention. | diff --git a/docs/design/multi-tier-testing-strategy.md b/docs/design/multi-tier-testing-strategy.md index 99b000b7f..1933a9fe6 100644 --- a/docs/design/multi-tier-testing-strategy.md +++ b/docs/design/multi-tier-testing-strategy.md @@ -862,12 +862,34 @@ The test must verify: 3. Image preprocessing config (resize, normalize, crop) matches HF 4. Tokenizer and processor files are present and well-formed -### Proposed test tiers for ORT GenAI +### ORT GenAI runtime tiers -| Level | What it tests | Cost | CI cadence | -|-------|--------------------------------------|----------|---------------| -| L1+ | Config generation (no weights) | Seconds | Every PR | -| E2E | Full pipeline: export + load + gen | Minutes | Weekly / GPU | +| Lane | What it tests | Runtime | CI cadence | +|------|---------------|---------|------------| +| Fast | Generated generic `decoder` package, tokenizer load, model creation, prefill, four cache-threaded decode steps, deterministic tokens, save/reload, and malformed config rejection | CPU; latest stable `onnxruntime-genai==0.15.2` | Relevant PRs, main, scheduled/manual | +| Real | YAML-enrolled immutable artifacts with exact tokenizer provenance, byte budget, deterministic full-length generation, and disposable per-test Hub/Xet caches | CPU; currently `onnxruntime-genai==0.15.2` | Weekly and manual | + +Run the lanes locally: + +```bash +python -m pytest tests/ort_genai_e2e_test.py -m ort_genai_fast -v +python -m pytest tests/gguf_small_model_runtime_integration_test.py \ + -m ort_genai_real -v +``` + +The `ort_genai` object in each golden-case YAML is the enrollment record. +Schema and coverage tests fail when a runtime-supported GGUF route lacks a +matching evidence ID. Released lanes use the architecture-neutral +`model.type=decoder`; main-only `state_groups` metadata is explicitly excluded. +Changes to graph construction, task wiring, ORT GenAI config generation, +tokenizer packaging, and GGUF qtype/runtime helpers conservatively select the +fast E2E lane through `detect_affected_models.py`. + +CUDA is not a required PR claim. The existing self-hosted GPU lane uses a +CUDA-12-specific pre-release ORT feed, so adding it to this stable-wheel +lane would conflate runtime and EP coverage. CUDA runtime E2E remains suitable +for scheduled/manual validation once that runner can install an equivalently +pinned released stack. --- diff --git a/pyproject.toml b/pyproject.toml index 5d1fb1562..c9096464b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,8 @@ markers = [ "arch_validation: L2 architecture validation tests that download real HF configs (no weights) and build full-size ONNX graphs (deselect with '-m \"not arch_validation\"')", "golden: L4 checkpoint-verified golden comparison tests (deselect with '-m \"not golden\"')", "generation: L5 generation end-to-end golden tests (deselect with '-m \"not generation\"')", + "ort_genai_fast: network-free end-to-end tests against released onnxruntime-genai wheels", + "ort_genai_real: pinned real-artifact onnxruntime-genai generation tests", ] [tool.mypy] diff --git a/scripts/detect_affected_models.py b/scripts/detect_affected_models.py index d575b9b49..94bd143d3 100644 --- a/scripts/detect_affected_models.py +++ b/scripts/detect_affected_models.py @@ -35,15 +35,33 @@ _SRC_ROOT = _PROJECT_ROOT / "src" / "mobius" # ---------------------------------------------------------------- -# Shared infrastructure paths — any change triggers run_all. -# Currently empty: core infra files (e.g. _configs.py, _registry.py) -# are NOT under _TRACEABLE_PREFIXES and therefore classify as "other", -# triggering no model tests. Add entries here to re-enable run_all for -# specific paths when broader regression coverage is needed. +# Shared graph/runtime infrastructure paths — any change triggers run_all. +# This is intentionally limited to surfaces that can alter exported model +# graphs, runtime metadata, tokenizer assets, or GGUF qtype handling. # ---------------------------------------------------------------- -_SHARED_INFRA_PATTERNS: tuple[str, ...] = () +_SHARED_INFRA_PATTERNS: tuple[str, ...] = ( + "src/mobius/integrations/gguf/_runtime_evidence.py", + "src/mobius/integrations/gguf/_runtime_package.py", + "src/mobius/integrations/gguf/_tokenizer.py", + "src/mobius/integrations/gguf/_builder.py", + "src/mobius/integrations/gguf/_quant_registry.py", + "src/mobius/integrations/gguf/_repacker.py", + "src/mobius/integrations/gguf/_reader.py", + "src/mobius/_builder.py", + "src/mobius/_model_package.py", + "src/mobius/_optimizations.py", + "src/mobius/_weight_loading.py", + "tests/ort_genai_e2e_test.py", + "tests/gguf_small_model_runtime_integration_test.py", + "testdata/cases/schema.json", + ".github/workflows/ort_genai_e2e.yml", + "pyproject.toml", +) -_SHARED_INFRA_PREFIXES: tuple[str, ...] = () +_SHARED_INFRA_PREFIXES: tuple[str, ...] = ( + "src/mobius/integrations/ort_genai/", + "src/mobius/tasks/", +) # Traceable infrastructure: component files that are analyzed via the # import graph to find which models they actually affect, rather than @@ -62,6 +80,11 @@ def classify_file(path: str) -> str: """ normalized = path.replace("\\", "/") + if normalized in _SHARED_INFRA_PATTERNS: + return "shared_infra" + if any(normalized.startswith(prefix) for prefix in _SHARED_INFRA_PREFIXES): + return "shared_infra" + if not normalized.startswith("src/mobius/"): # Test infrastructure files that affect all models if normalized == "tests/conftest.py": diff --git a/scripts/detect_affected_models_test.py b/scripts/detect_affected_models_test.py index 5c5d2795d..c60530693 100644 --- a/scripts/detect_affected_models_test.py +++ b/scripts/detect_affected_models_test.py @@ -49,7 +49,7 @@ def test_component_file(self): assert classify_file("src/mobius/components/_attention.py") == "traceable" def test_task_file(self): - assert classify_file("src/mobius/tasks/_causal_lm.py") == "traceable" + assert classify_file("src/mobius/tasks/_causal_lm.py") == "shared_infra" def test_configs_file(self): assert classify_file("src/mobius/_configs.py") == "other" @@ -58,7 +58,7 @@ def test_registry_file(self): assert classify_file("src/mobius/_registry.py") == "other" def test_builder_file(self): - assert classify_file("src/mobius/_builder.py") == "other" + assert classify_file("src/mobius/_builder.py") == "shared_infra" def test_exporter_file(self): assert classify_file("src/mobius/_exporter.py") == "other" @@ -85,7 +85,15 @@ def test_readme(self): assert classify_file("README.md") == "other" def test_pyproject(self): - assert classify_file("pyproject.toml") == "other" + assert classify_file("pyproject.toml") == "shared_infra" + + def test_ort_genai_integration_file(self): + assert ( + classify_file("src/mobius/integrations/ort_genai/ep_config.py") == "shared_infra" + ) + + def test_ort_genai_workflow(self): + assert classify_file(".github/workflows/ort_genai_e2e.yml") == "shared_infra" def test_windows_paths(self): assert classify_file("src\\mobius\\models\\falcon.py") == "model" @@ -224,16 +232,19 @@ def test_component_change_traces_affected_models(self): # _attention.py is imported by many models — should find affected types assert len(result["affected"]) > 0 - def test_task_change_does_not_trigger_run_all(self): - """Task files are traceable but produce an empty affected set. - - No model imports ``mobius.tasks`` directly (tasks are looked up at - runtime by string keys), so tracing through the import graph finds - no dependents. Documented limitation — see PR description. - """ + def test_task_change_triggers_runtime_matrix(self): result = detect_affected_models(["src/mobius/tasks/_causal_lm.py"]) - assert result["run_all"] is False - assert result["affected"] == [] + assert result == {"affected": [], "run_all": True} + + def test_ort_genai_change_triggers_runtime_matrix(self): + result = detect_affected_models( + ["src/mobius/integrations/ort_genai/_execution_providers.py"] + ) + assert result == {"affected": [], "run_all": True} + + def test_dependency_change_triggers_runtime_matrix(self): + result = detect_affected_models(["pyproject.toml"]) + assert result == {"affected": [], "run_all": True} def test_configs_change_no_run_all(self): """_configs.py no longer triggers run_all (shared_infra disabled).""" @@ -253,7 +264,7 @@ def test_test_configs_with_model_uses_model_scope(self): ] ) assert result["run_all"] is False - assert result["affected"] == ["lfm2"] + assert result["affected"] == ["lfm2", "lfm2_moe", "lfm2_vl"] def test_test_configs_with_unmapped_task_still_runs_all(self): result = detect_affected_models( @@ -348,18 +359,23 @@ def test_component_common_affects_many_models(self): # _common.py defines Linear, Embedding, LayerNorm — used everywhere assert len(result["affected"]) > 10 - def test_former_shared_infra_no_run_all(self): - """Former shared_infra files no longer trigger run_all.""" + def test_runtime_shared_infra_runs_all(self): for path in [ - "src/mobius/_configs.py", - "src/mobius/_registry.py", + "src/mobius/_model_package.py", + "src/mobius/integrations/ort_genai/auto_export.py", + "src/mobius/integrations/ort_genai/genai_config.py", + "src/mobius/integrations/gguf/_runtime_package.py", + "src/mobius/integrations/gguf/_tokenizer.py", + "src/mobius/integrations/gguf/_builder.py", + "src/mobius/integrations/gguf/_quant_registry.py", + "src/mobius/integrations/gguf/_repacker.py", "src/mobius/_builder.py", + "src/mobius/_optimizations.py", "src/mobius/_weight_loading.py", - "src/mobius/_model_package.py", - "src/mobius/models/__init__.py", + "testdata/cases/schema.json", ]: result = detect_affected_models([path]) - assert result["run_all"] is False, f"{path} should NOT trigger run_all" + assert result["run_all"] is True, f"{path} should trigger run_all" def test_traceable_and_model_combined(self): """A component + model file change returns union of affected types.""" diff --git a/src/mobius/_testing/golden.py b/src/mobius/_testing/golden.py index ded7ab42b..bdca71cad 100644 --- a/src/mobius/_testing/golden.py +++ b/src/mobius/_testing/golden.py @@ -139,6 +139,9 @@ class GoldenTestCase: tokenizer/reference checkpoint used to create the golden data. """ + ort_genai: dict[str, object] | None + """Pinned ORT GenAI E2E enrollment and released-runtime capability record.""" + yaml_path: Path """Absolute path to the source YAML file.""" @@ -267,6 +270,7 @@ def load_test_case(yaml_path: Path) -> GoldenTestCase: min_token_match_ratio=data.get("min_token_match_ratio"), architecture=data.get("architecture"), gguf_source=data.get("gguf"), + ort_genai=data.get("ort_genai"), yaml_path=yaml_path, ) diff --git a/src/mobius/integrations/gguf/_arch_registry.py b/src/mobius/integrations/gguf/_arch_registry.py index 66a0f9e81..ab00ff4b1 100644 --- a/src/mobius/integrations/gguf/_arch_registry.py +++ b/src/mobius/integrations/gguf/_arch_registry.py @@ -680,11 +680,15 @@ tensor_processor="llama", llama_qk_permute=True, runtime=Support.SUPPORTED, - runtime_evidence_ids=("smollm-135m-f16-onnxruntime-1.29.0",), + runtime_evidence_ids=( + "smollm-135m-f16-onnxruntime-1.29.0", + "smollm-135m-f16-ort-genai-0.15.2", + ), reason=( "Runtime support is restricted to exact structured evidence matches. Currently " "that is only neopolita/smollm-135m-gguf F16 at the pinned artifact, CPU import " - "route, ONNX Runtime 1.29.0, and HuggingFaceTB/SmolLM-135M tokenizer revision." + "route, evidenced ONNX Runtime/ORT GenAI versions, and the pinned " + "HuggingFaceTB/SmolLM-135M tokenizer revision." ), ), GGUFArchitectureSpec( diff --git a/src/mobius/integrations/gguf/_docs_test.py b/src/mobius/integrations/gguf/_docs_test.py index 517aceff0..56b63f7bd 100644 --- a/src/mobius/integrations/gguf/_docs_test.py +++ b/src/mobius/integrations/gguf/_docs_test.py @@ -50,7 +50,13 @@ def test_generated_census_counts_and_pin_are_closed() -> None: def test_runtime_support_requires_structured_evidence() -> None: supported = [spec for spec in iter_arch_specs() if spec.runtime is Support.SUPPORTED] assert [(spec.gguf_arch, spec.runtime_evidence_ids) for spec in supported] == [ - ("llama", ("smollm-135m-f16-onnxruntime-1.29.0",)) + ( + "llama", + ( + "smollm-135m-f16-onnxruntime-1.29.0", + "smollm-135m-f16-ort-genai-0.15.2", + ), + ) ] pins = {pin.artifact_id for pin in MMPROJ_ARTIFACT_PINS} diff --git a/src/mobius/integrations/gguf/_runtime_evidence.py b/src/mobius/integrations/gguf/_runtime_evidence.py index 4b1decb0d..b9e5106db 100644 --- a/src/mobius/integrations/gguf/_runtime_evidence.py +++ b/src/mobius/integrations/gguf/_runtime_evidence.py @@ -174,7 +174,7 @@ def _is_hex(value: str) -> bool: _SMOLLM_F16_ROUTE = ( '{"architecture":"llama","config_sha256":' - '"134f95e6a635d978737d712ed61ac8959acebdf080eafae838cf97f12c416430",' + '"b999a219ddf6046f1b6e5c082dab2d00b7ba1682169914430b1a3672fc735498",' '"execution_provider":"cpu","model_type":"llama","module_type":"llama",' '"preserve_quantization":false,"registry_import":{"config_key_map":null,' '"config_postprocessor":null,"llama_qk_permute":true,"offset_norm":false,' @@ -249,8 +249,31 @@ def _is_hex(value: str) -> bool: runtime_version="1.29.0", ) +_SMOLLM_F16_ORT_GENAI = dataclasses.replace( + _SMOLLM_F16_ONNX_RUNTIME, + evidence_id="smollm-135m-f16-ort-genai-0.15.2", + runtime_package_files=( + "genai_config.json", + "gguf_tokenizer_manifest.json", + "model.onnx", + "model.onnx.data", + "runtime_compatibility.json", + "special_tokens_map.json", + "tokenizer.json", + "tokenizer_config.json", + ), + runtime_package_sha256="43568320f669d259d5a570ee04bd6378316ab31ce2fcb6383e75b479b4f2b349", + deterministic_test="test_smollm_generic_ort_genai_generation", + stateful_semantics="ORT GenAI prefill plus 20 cache-threaded decode steps", + runtime="ort-genai", + runtime_version="0.15.2", +) + _RUNTIME_EVIDENCE: MappingProxyType[str, GGUFRuntimeEvidence] = MappingProxyType( - {_SMOLLM_F16_ONNX_RUNTIME.evidence_id: _SMOLLM_F16_ONNX_RUNTIME} + { + record.evidence_id: record + for record in (_SMOLLM_F16_ONNX_RUNTIME, _SMOLLM_F16_ORT_GENAI) + } ) diff --git a/testdata/cases/causal-lm/smollm-135m-gguf-f16.yaml b/testdata/cases/causal-lm/smollm-135m-gguf-f16.yaml index 0dba9a6be..ffadaf3d2 100644 --- a/testdata/cases/causal-lm/smollm-135m-gguf-f16.yaml +++ b/testdata/cases/causal-lm/smollm-135m-gguf-f16.yaml @@ -15,7 +15,7 @@ gguf: F16: 211 F32: 61 execution_provider: "cpu" - config_sha256: "134f95e6a635d978737d712ed61ac8959acebdf080eafae838cf97f12c416430" + config_sha256: "b999a219ddf6046f1b6e5c082dab2d00b7ba1682169914430b1a3672fc735498" preserve_quantization: false keep_quantized: true tokenizer: @@ -34,6 +34,18 @@ gguf: size: 3685 sha256: "238ad6b60d48e471624ea70bc79e92f2611844d5016471fee8c167854bcb98e8" +ort_genai: + tier: "real" + runtime_versions: ["0.15.2"] + model_type: "decoder" + runtime_evidence_id: "smollm-135m-f16-ort-genai-0.15.2" + execution_provider: "cpu" + max_download_bytes: 300000000 + released_capabilities: + "0.15.2": + generic_decoder: true + state_groups: false + inputs: prompts: - "Once upon a time," diff --git a/testdata/cases/schema.json b/testdata/cases/schema.json index 53c259096..95bc6f803 100644 --- a/testdata/cases/schema.json +++ b/testdata/cases/schema.json @@ -262,6 +262,70 @@ }, "description": "Pinned GGUF import source. The top-level model_id remains the independent HuggingFace reference." }, + "ort_genai": { + "type": "object", + "additionalProperties": false, + "required": [ + "tier", + "runtime_versions", + "model_type", + "runtime_evidence_id", + "execution_provider", + "max_download_bytes", + "released_capabilities" + ], + "properties": { + "tier": { + "const": "real", + "description": "Enroll this case in the pinned real-artifact ORT GenAI lane." + }, + "runtime_versions": { + "type": "array", + "items": { + "enum": [ + "0.15.2" + ] + }, + "minItems": 1, + "maxItems": 1, + "uniqueItems": true, + "description": "Released onnxruntime-genai versions validated for this route." + }, + "model_type": { + "type": "string", + "minLength": 1, + "description": "Expected model.type in the generated genai_config.json." + }, + "runtime_evidence_id": { + "type": "string", + "minLength": 1, + "description": "Structured runtime evidence record proving this exact artifact route." + }, + "execution_provider": { + "const": "cpu", + "description": "Required CI currently makes only a CPU execution-provider claim." + }, + "max_download_bytes": { + "type": "integer", + "minimum": 1, + "maximum": 536870912, + "description": "Hard per-case aggregate artifact download budget." + }, + "released_capabilities": { + "type": "object", + "additionalProperties": false, + "required": [ + "0.15.2" + ], + "properties": { + "0.15.2": { + "$ref": "#/$defs/ortGenaiCapabilities" + } + }, + "description": "Explicit released-runtime capability expectations." + } + } + }, "level": { "type": "string", "enum": [ @@ -446,9 +510,27 @@ }, "additionalProperties": false }, - "architecture": { - "type": "string", - "description": "Optional registry architecture key (e.g. Qwen35MtpModel) forcing a specific module class + task at build time. Needed for auxiliary heads (DFlash, MTP) that share a base checkpoint whose architectures field would otherwise auto-route to the base model." + "architecture": { + "type": "string", + "description": "Optional registry architecture key (e.g. Qwen35MtpModel) forcing a specific module class + task at build time. Needed for auxiliary heads (DFlash, MTP) that share a base checkpoint whose architectures field would otherwise auto-route to the base model." + } +}, +"$defs": { + "ortGenaiCapabilities": { + "type": "object", + "additionalProperties": false, + "required": [ + "generic_decoder", + "state_groups" + ], + "properties": { + "generic_decoder": { + "const": true + }, + "state_groups": { + "const": false + } } +} } } diff --git a/tests/gguf_small_model_runtime_integration_test.py b/tests/gguf_small_model_runtime_integration_test.py index e78be7ea2..f79d4a89f 100644 --- a/tests/gguf_small_model_runtime_integration_test.py +++ b/tests/gguf_small_model_runtime_integration_test.py @@ -15,16 +15,21 @@ import hashlib import json +import os +import shutil from collections import Counter from dataclasses import dataclass +from importlib.metadata import version from pathlib import Path from unittest import mock +import huggingface_hub.constants import numpy as np import onnxruntime as ort import pytest import torch -from huggingface_hub import hf_hub_download +import yaml +from huggingface_hub import get_hf_file_metadata, hf_hub_download, hf_hub_url from transformers import AutoModelForCausalLM, AutoTokenizer from mobius import ModelPackage, build_from_gguf @@ -72,7 +77,7 @@ class _RuntimeCase: reference_revision="1d461723eec654e65efdc40cf49301c89c0c92f4", prompt="Once upon a time,", tensor_qtypes={"F16": 211, "F32": 61}, - config_sha256="134f95e6a635d978737d712ed61ac8959acebdf080eafae838cf97f12c416430", + config_sha256="b999a219ddf6046f1b6e5c082dab2d00b7ba1682169914430b1a3672fc735498", generated_tokens=( 665, 436, @@ -218,6 +223,27 @@ class _RuntimeCase: ) +@pytest.fixture +def isolated_hf_cache(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + """Keep each real-artifact test's Hub/Xet state isolated and disposable.""" + cache_root = tmp_path / "huggingface-cache" + hub_cache = cache_root / "hub" + xet_cache = cache_root / "xet" + cache_root.mkdir() + monkeypatch.setenv("HF_HOME", str(cache_root)) + monkeypatch.setenv("HF_HUB_CACHE", str(hub_cache)) + monkeypatch.setenv("HF_XET_CACHE", str(xet_cache)) + monkeypatch.setenv("TRANSFORMERS_CACHE", str(cache_root / "transformers")) + monkeypatch.setattr(huggingface_hub.constants, "HF_HOME", str(cache_root)) + monkeypatch.setattr(huggingface_hub.constants, "HF_HUB_CACHE", str(hub_cache)) + monkeypatch.setattr(huggingface_hub.constants, "HUGGINGFACE_HUB_CACHE", str(hub_cache)) + monkeypatch.setattr(huggingface_hub.constants, "HF_XET_CACHE", str(xet_cache)) + try: + yield cache_root + finally: + shutil.rmtree(cache_root, ignore_errors=True) + + def _sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as stream: @@ -633,14 +659,49 @@ def capture_save(package: ModelPackage, *args: object, **kwargs: object) -> None @pytest.mark.integration @pytest.mark.integration_slow -def test_smollm_generic_ort_genai_generation(tmp_path: Path) -> None: +@pytest.mark.ort_genai_real +def test_smollm_generic_ort_genai_generation( + tmp_path: Path, + isolated_hf_cache: Path, +) -> None: """The one evidenced GGUF route loads through ORT GenAI's generic decoder.""" - from importlib.metadata import version - - ort_genai = pytest.importorskip("onnxruntime_genai") - from mobius.integrations.ort_genai import write_ort_genai_config + import onnxruntime_genai as ort_genai case = _CASES[0] + case_yaml = Path("testdata/cases/causal-lm/smollm-135m-gguf-f16.yaml") + metadata = yaml.safe_load(case_yaml.read_text(encoding="utf-8"))["ort_genai"] + installed_version = version("onnxruntime-genai") + expected_version = os.environ.get("MOBIUS_EXPECTED_ORT_GENAI_VERSION") + if expected_version: + assert installed_version == expected_version + assert installed_version in metadata["runtime_versions"] + assert metadata["released_capabilities"][installed_version] == { + "generic_decoder": True, + "state_groups": False, + } + download_specs = [ + ( + case.gguf_repository, + case.gguf_revision, + case.gguf_filename, + case.gguf_size, + ), + *( + (case.tokenizer_repository, case.tokenizer_revision, filename, size) + for filename, size, _ in case.tokenizer_assets + ), + ] + remote_download_bytes = 0 + for repository, revision, filename, expected_size in download_specs: + remote = get_hf_file_metadata( + hf_hub_url(repository, filename, revision=revision), + timeout=30, + ) + assert remote.commit_hash == revision + assert remote.size == expected_size + remote_download_bytes += remote.size + assert remote_download_bytes <= metadata["max_download_bytes"] + assert isolated_hf_cache.exists() gguf_path = Path( hf_hub_download( repo_id=case.gguf_repository, @@ -648,38 +709,43 @@ def test_smollm_generic_ort_genai_generation(tmp_path: Path) -> None: filename=case.gguf_filename, ) ) - output_dir = tmp_path / "smollm-ort-genai" - captured: list[ModelPackage] = [] - original_save = ModelPackage.save - - def capture_save(package: ModelPackage, *args: object, **kwargs: object) -> None: - captured.append(package) - original_save(package, *args, **kwargs) - - with mock.patch.object(ModelPackage, "save", capture_save): - main( - [ - "build-gguf", - str(gguf_path), - "--output", - str(output_dir), - "--dtype", - "f32", - "--execution-provider", - "cpu", - ] + assert gguf_path.stat().st_size == case.gguf_size + assert _sha256(gguf_path) == case.gguf_sha256 + for filename, size, sha256 in case.tokenizer_assets: + asset_path = Path( + hf_hub_download( + repo_id=case.tokenizer_repository, + revision=case.tokenizer_revision, + filename=filename, + ) ) - - assert len(captured) == 1 - write_ort_genai_config( - captured[0], - str(output_dir), - hf_model_id=case.tokenizer_repository, - revision=case.tokenizer_revision, - runtime_version=version("onnxruntime-genai"), + assert asset_path.stat().st_size == size + assert _sha256(asset_path) == sha256 + output_dir = tmp_path / "smollm-ort-genai" + main( + [ + "build-gguf", + str(gguf_path), + "--output", + str(output_dir), + "--dtype", + "f32", + "--execution-provider", + "cpu", + "--runtime", + "ort-genai", + "--runtime-version", + installed_version, + "--tokenizer-repository", + case.tokenizer_repository, + "--tokenizer-revision", + case.tokenizer_revision, + "--local-files-only", + ] ) config = json.loads((output_dir / "genai_config.json").read_text()) - assert config["model"]["type"] == "decoder" + assert config["model"]["type"] == metadata["model_type"] + assert "state_groups" not in config["model"]["decoder"] model = ort_genai.Model(str(output_dir)) tokenizer = ort_genai.Tokenizer(model) @@ -697,4 +763,5 @@ def capture_save(package: ModelPackage, *args: object, **kwargs: object) -> None generator.generate_next_token() generated.append(generator.get_next_tokens()[0]) + assert len(generated) == len(case.generated_tokens) assert generated == list(case.generated_tokens) diff --git a/tests/ort_genai_e2e_test.py b/tests/ort_genai_e2e_test.py new file mode 100644 index 000000000..1797df83a --- /dev/null +++ b/tests/ort_genai_e2e_test.py @@ -0,0 +1,271 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Fast, network-free end-to-end tests for released ONNX Runtime GenAI wheels.""" + +from __future__ import annotations + +import dataclasses +import gc +import importlib +import json +import os +from importlib.metadata import version +from pathlib import Path +from typing import Any + +import onnx_ir as ir +import pytest +from onnxscript import GraphBuilder + +from mobius._model_package import ModelPackage +from mobius.integrations.ort_genai import write_ort_genai_config + +_EXPECTED_ORT_GENAI_VERSION = "0.15.2" + + +@dataclasses.dataclass +class _SyntheticConfig: + model_type: str = "synthetic_ci_decoder" + vocab_size: int = 8 + hidden_size: int = 8 + num_hidden_layers: int = 1 + num_attention_heads: int = 2 + num_key_value_heads: int = 2 + head_dim: int = 4 + max_position_embeddings: int = 32 + bos_token_id: int = 1 + eos_token_id: int = 7 + pad_token_id: int = 6 + + +def _typed(value: ir.Value, dtype: ir.DataType, shape: list[int | str]) -> ir.Value: + value.type = ir.TensorType(dtype) + value.shape = ir.Shape(shape) + return value + + +def _synthetic_decoder_package() -> ModelPackage: + """Build a decoder whose next token proves the prior KV cache was threaded.""" + graph = ir.Graph([], [], nodes=[], name="decoder", opset_imports={"": 23}) + builder = GraphBuilder(graph) + input_ids = builder.input("input_ids", ir.DataType.INT64, ["batch", "sequence"]) + builder.input("attention_mask", ir.DataType.INT64, ["batch", "total_sequence"]) + builder.input("position_ids", ir.DataType.INT64, ["batch", "sequence"]) + past_key = builder.input( + "past_key_values.0.key", + ir.DataType.FLOAT, + ["batch", 2, "past_sequence", 4], + ) + past_value = builder.input( + "past_key_values.0.value", + ir.DataType.FLOAT, + ["batch", 2, "past_sequence", 4], + ) + + token_shape = builder.op.Shape(input_ids) + past_sequence = builder.op.Shape(past_key, start=2, end=3) + token_id = builder.op.Cast(past_sequence, to=ir.DataType.INT64) + one_hot = builder.op.OneHot( + token_id, + builder.op.Constant(value_int=8), + builder.op.Constant(value_floats=[0.0, 1.0]), + axis=-1, + ) + logits = builder.op.Expand( + builder.op.Unsqueeze(one_hot, builder.op.Constant(value_ints=[0])), + builder.op.Concat(token_shape, builder.op.Constant(value_ints=[8]), axis=0), + ) + batch = builder.op.Shape(input_ids, start=0, end=1) + sequence = builder.op.Shape(input_ids, start=1, end=2) + cache_update_shape = builder.op.Concat( + batch, + builder.op.Constant(value_ints=[2]), + sequence, + builder.op.Constant(value_ints=[4]), + axis=0, + ) + cache_update = builder.op.ConstantOfShape( + cache_update_shape, + value=ir.tensor([0.0]), + ) + present_key = builder.op.Concat(past_key, cache_update, axis=2) + present_value = builder.op.Concat(past_value, cache_update, axis=2) + + builder.add_output( + _typed(logits, ir.DataType.FLOAT, ["batch", "sequence", 8]), + "logits", + ) + builder.add_output( + _typed(present_key, ir.DataType.FLOAT, ["batch", 2, "present_sequence", 4]), + "present.0.key", + ) + builder.add_output( + _typed(present_value, ir.DataType.FLOAT, ["batch", 2, "present_sequence", 4]), + "present.0.value", + ) + return ModelPackage( + {"model": ir.Model(graph, ir_version=10)}, + config=_SyntheticConfig(), + ) + + +def _write_tokenizer(output_dir: Path) -> None: + vocabulary = { + "[UNK]": 0, + "[BOS]": 1, + "hello": 2, + "world": 3, + "!": 4, + "unused": 5, + "[PAD]": 6, + "[EOS]": 7, + } + tokenizer = { + "version": "1.0", + "truncation": None, + "padding": None, + "added_tokens": [], + "normalizer": {"type": "Lowercase"}, + "pre_tokenizer": {"type": "Whitespace"}, + "post_processor": None, + "decoder": None, + "model": { + "type": "BPE", + "dropout": None, + "unk_token": "[UNK]", + "continuing_subword_prefix": "", + "end_of_word_suffix": "", + "fuse_unk": False, + "byte_fallback": False, + "ignore_merges": False, + "vocab": vocabulary, + "merges": [], + }, + } + (output_dir / "tokenizer.json").write_text( + json.dumps(tokenizer, indent=2) + "\n", + encoding="utf-8", + ) + (output_dir / "tokenizer_config.json").write_text( + json.dumps( + { + "tokenizer_class": "LlamaTokenizer", + "bos_token": "[BOS]", + "eos_token": "[EOS]", + "pad_token": "[PAD]", + "unk_token": "[UNK]", + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + + +@pytest.fixture(scope="module") +def ort_genai_module() -> Any: + """Import the selected lane's required runtime only when these tests execute.""" + return importlib.import_module("onnxruntime_genai") + + +def _generate( + ort_genai: Any, + package_dir: Path, + prompt: str, + max_new_tokens: int, +) -> tuple[list[int], str]: + model = ort_genai.Model(str(package_dir)) + tokenizer = ort_genai.Tokenizer(model) + prompt_ids = tokenizer.encode(prompt) + params = ort_genai.GeneratorParams(model) + params.set_search_options( + max_length=len(prompt_ids) + max_new_tokens, + do_sample=False, + ) + generator = ort_genai.Generator(model, params) + generator.append_tokens(prompt_ids) + + generated: list[int] = [] + for _ in range(max_new_tokens): + assert not generator.is_done() + generator.generate_next_token() + generated.append(generator.get_next_tokens()[0]) + decoded = tokenizer.decode(prompt_ids) + del generator, tokenizer, model + gc.collect() + return generated, decoded + + +@pytest.mark.integration +@pytest.mark.ort_genai_fast +def test_generic_decoder_end_to_end_and_reload( + tmp_path: Path, + ort_genai_module: Any, +) -> None: + """Exercise tokenizer, prefill, threaded KV decode, and package reload.""" + expected_version = os.environ.get("MOBIUS_EXPECTED_ORT_GENAI_VERSION") + installed_version = version("onnxruntime-genai") + if expected_version: + assert installed_version == expected_version + assert installed_version == _EXPECTED_ORT_GENAI_VERSION + + package_dir = tmp_path / "synthetic-decoder" + package = _synthetic_decoder_package() + package.save(package_dir) + _write_tokenizer(package_dir) + write_ort_genai_config( + package, + str(package_dir), + runtime_version=installed_version, + ) + + config = json.loads((package_dir / "genai_config.json").read_text(encoding="utf-8")) + assert config["model"]["type"] == "decoder" + assert "state_groups" not in config["model"]["decoder"] + + # Prefill sees an empty cache; each decode step must receive the prior + # present cache for the selected token to advance. + expected_tokens = [0, 1, 2, 3] + first_tokens, decoded_prompt = _generate( + ort_genai_module, + package_dir, + "hello", + len(expected_tokens), + ) + assert decoded_prompt == "hello" + assert len(first_tokens) == len(expected_tokens) + assert first_tokens == expected_tokens + + reloaded_tokens, _ = _generate( + ort_genai_module, + package_dir, + "hello", + len(expected_tokens), + ) + assert len(reloaded_tokens) == len(expected_tokens) + assert reloaded_tokens == expected_tokens + + +@pytest.mark.integration +@pytest.mark.ort_genai_fast +def test_malformed_genai_config_is_rejected( + tmp_path: Path, + ort_genai_module: Any, +) -> None: + package_dir = tmp_path / "malformed" + package = _synthetic_decoder_package() + package.save(package_dir) + _write_tokenizer(package_dir) + write_ort_genai_config( + package, + str(package_dir), + runtime_version=version("onnxruntime-genai"), + ) + config_path = package_dir / "genai_config.json" + config = json.loads(config_path.read_text(encoding="utf-8")) + del config["model"]["decoder"]["filename"] + config_path.write_text(json.dumps(config), encoding="utf-8") + + with pytest.raises(RuntimeError, match=r"filename|model path is a directory"): + ort_genai_module.Model(str(package_dir)) diff --git a/tests/yaml_schema_test.py b/tests/yaml_schema_test.py index 8978992fa..76cc0ba82 100644 --- a/tests/yaml_schema_test.py +++ b/tests/yaml_schema_test.py @@ -22,6 +22,10 @@ import pytest import yaml +from mobius.integrations.gguf._arch_registry import iter_arch_specs +from mobius.integrations.gguf._runtime_evidence import runtime_evidence +from mobius.integrations.gguf._spec import Support + _REPO_ROOT = Path(__file__).resolve().parent.parent _CASES_DIR = _REPO_ROOT / "testdata" / "cases" _SCHEMA_PATH = _CASES_DIR / "schema.json" @@ -108,3 +112,51 @@ def test_all_yaml_task_types_are_in_schema() -> None: "Add them to the task_type enum in testdata/cases/schema.json:\n" + "\n".join(missing) ) + + +def test_every_runtime_supported_route_has_ort_genai_e2e_enrollment() -> None: + """Runtime support cannot outgrow pinned downstream generation coverage.""" + enrolled: dict[str, dict[str, Any]] = {} + for yaml_path in _YAML_FILES: + data = yaml.safe_load(yaml_path.read_text(encoding="utf-8")) + marker = data.get("ort_genai") + if marker: + evidence_id = marker["runtime_evidence_id"] + assert evidence_id not in enrolled, ( + f"Duplicate ORT GenAI evidence ID: {evidence_id}" + ) + enrolled[evidence_id] = marker + + required_routes = { + ( + evidence.architecture, + evidence.repository, + evidence.revision, + evidence.filename, + evidence.import_route, + ) + for spec in iter_arch_specs() + if spec.runtime is Support.SUPPORTED + for evidence_id in spec.runtime_evidence_ids + if (evidence := runtime_evidence(evidence_id)) is not None + } + enrolled_routes = set() + for evidence_id, marker in enrolled.items(): + evidence = runtime_evidence(evidence_id) + assert evidence is not None + assert evidence.runtime == "ort-genai" + assert evidence.runtime_version in marker["runtime_versions"] + enrolled_routes.add( + ( + evidence.architecture, + evidence.repository, + evidence.revision, + evidence.filename, + evidence.import_route, + ) + ) + assert marker["execution_provider"] == "cpu" + assert marker["model_type"] == "decoder" + assert marker["runtime_versions"] == ["0.15.2"] + assert marker["max_download_bytes"] >= evidence.size + assert enrolled_routes == required_routes