diff --git a/docs/api/build_from_gguf.md b/docs/api/build_from_gguf.md index bd008dae6..106e182b8 100644 --- a/docs/api/build_from_gguf.md +++ b/docs/api/build_from_gguf.md @@ -15,12 +15,12 @@ from mobius import build_from_gguf | Census | Total | Closure | |---|---:|---| -| Architectures | 147 | graph verdicts: {'deferred': 88, 'rejected': 2, 'supported': 57}; importable: 55; quantized import: {'rejected': 11, 'supported': 136}; runtime: {'deferred': 145, 'rejected': 2} | +| Architectures | 147 | graph verdicts: {'deferred': 88, 'rejected': 2, 'supported': 57}; importable: 55; quantized import: {'rejected': 11, 'supported': 136}; runtime: {'deferred': 144, 'rejected': 2, 'supported': 1} | | Active stored qtypes | 25 | 24 have an import route; 1 are explicitly deferred with no route | | Serialized projector strings | 60 | {'graph-importable': 2, 'runtime-supported': 0} | -| Tokenizer pre identifiers | 87 | 56 semantic groups; all default to deferred and become exact-copy only with a validated embedded `tokenizer.huggingface.json` | +| Tokenizer pre identifiers | 87 | 56 semantic groups; all default to deferred and become materializable only from a validated embedded `tokenizer.huggingface.json` or an exact pinned source in runtime evidence | -`SUPPORTED` means the named capability is implemented and mechanically tested. `DEFERRED` means it is intentionally unavailable pending the stated work. `REJECTED` means the input or route is invalid by policy. Graph support proves construction/execution only; runtime support additionally requires a pinned real artifact, independent parity, and deterministic generation or stateful semantics. Tokenizer `copy` delegates algorithm semantics to an embedded, vocabulary-identical tokenizer JSON; it is not a reconstructed or independently proven tokenizer. +`SUPPORTED` means the named capability is implemented and mechanically tested. `DEFERRED` means it is intentionally unavailable pending the stated work. `REJECTED` means the input or route is invalid by policy. Graph support proves construction/execution only; runtime support additionally requires a pinned real artifact, independent parity, and deterministic generation or stateful semantics. Tokenizer `copy` delegates algorithm semantics to an embedded, vocabulary-identical tokenizer JSON. A `pinned-source` route additionally binds an immutable Hub revision, exact asset hashes, and all reconstructible GGUF tokenizer semantics. @@ -31,21 +31,20 @@ normally stores only an opaque `tokenizer.ggml.pre` identifier, not the full normalizer, pre-tokenizer, added-token, decoder, and post-processor pipeline. Mobius never substitutes a generic BPE/SentencePiece tokenizer. -The only materialized route is `copy`: the GGUF must contain -`tokenizer.huggingface.json`, and its loadable ordered vocabulary must exactly -match `tokenizer.ggml.tokens`. Mobius copies that tokenizer pipeline verbatim -rather than claiming to reconstruct its opaque `pre` semantics, then writes -`tokenizer_config.json`, `special_tokens_map.json`, and a provenance manifest. -Otherwise the route is `deferred`: graph-only import remains available, while -runtime packaging rejects incomplete metadata before durable output. Unknown -identifiers and malformed or contradictory complete tokenizer tables reject -before graph construction. +The `copy` route requires `tokenizer.huggingface.json` in the GGUF and an exact +ordered-vocabulary match. The `pinned-source` route is narrower: a runtime evidence +record names one immutable Hub revision and every copied asset's size and SHA-256. +Mobius additionally compares ordered token IDs, merge order, special-token IDs, +flags, and chat templates against all corresponding GGUF metadata. The remaining +normalizer, pre-tokenizer, decoder, and post-processor semantics are accepted only +through those exact asset hashes, never reconstructed from `tokenizer.ggml.pre`. +Any missing or contradictory field rejects before durable output. The graph package records a canonical digest of every tokenizer metadata field. Runtime packaging rechecks that digest before writing, so replacing a local GGUF between graph construction and package emission cannot mix tokenizer -identity. The manifest reports ORT tokenizer compatibility as delegated to the -embedded tokenizer JSON rather than claiming independent algorithm parity. +identity. The manifest records the selected route, immutable source revision, +asset hashes, and GGUF tokenizer metadata hash. This generated policy table is pinned to llama.cpp commit `8d9af256337d1a501250f9bbf4c0859a654bddd6`. It enumerates all 87 accepted @@ -305,13 +304,11 @@ validated embedding owner, but Mobius never invents a second head or silently drops an explicit conflicting output. Complete runtime packages are staged in a sibling temporary directory and -replace the destination with rollback on an ordinary in-process publication -failure. Replacement uses separate destination-to-backup and stage-to-destination -renames: readers can briefly observe no destination, and a process or host crash -between them requires manual recovery from the sibling backup. This is not a -single-step atomic exchange, crash-durable transaction, or multi-process lock; -callers must serialize concurrent writers. The source tokenizer digest is -rechecked before publication. Publication also requires the exact runtime +published with one atomic rename. An existing destination is rejected rather +than moved aside, so readers never observe a replacement gap or mixed package. +This is not a multi-process lock; callers racing to the same absent destination +must handle one publication failure. The source tokenizer digest is rechecked +before publication. Publication also requires the exact runtime version from the evidence record and rechecks the source filename, size, SHA-256, canonical architecture, tensor/qtype census, and complete graph-shaping import route captured during construction. The serialized ONNX file list and graph @@ -421,7 +418,7 @@ before graph construction or durable output. | `lfm2moe` | — | model=`lfm2_moe`; tensor=`lfm2`+`lfm2_moe_extras` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=rejected | Config extraction, exact pinned tensor-name closure, GGUF value transforms, and synthetic recurrent-state execution are covered, but no representative real-weight GGUF has yet passed independent full-logit parity and deterministic multi-token stateful ORT generation. Runtime packaging remains deferred until that evidence exists. The mobius graph uses floating Linear modules for this architecture, so no MatMulNBits or BlockQuantizedMatMul target can consume preserved GGUF projection weights. Use keep_quantized=False for explicit float import. | | `llada` | — | model=`llada`; tensor=`llama` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=supported | Config extraction, suffix-exact tensor closure, masked-diffusion task dispatch, and synthetic full-sequence execution are covered, but no pinned real GGUF has passed independent Hugging Face/llama.cpp masked-step logit parity and deterministic multi-step generation parity. Runtime packaging remains deferred until both exist. | | `llada-moe` | — | model=`llada`; module=`llada_moe`; tensor=`llama`+`diffusion_fused_qkv`+`moe_qk_norm_extras`+`moe_extras` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=supported | Config extraction, suffix-exact tensor closure, masked-diffusion task dispatch, and synthetic full-sequence execution are covered, but no pinned real GGUF has passed independent Hugging Face/llama.cpp masked-step logit parity and deterministic multi-step generation parity. Runtime packaging remains deferred until both exist. | -| `llama` | `mistral` | model=`llama`; tensor=`llama` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=supported | Config extraction, exact tensor-name closure, and a full synthetic GGUF graph build are covered, but no representative real-weight GGUF has yet passed ORT parity or generation validation. Runtime packaging remains deferred until that evidence exists. | +| `llama` | `mistral` | model=`llama`; tensor=`llama` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=supported; quantized_import=supported | Runtime support is restricted to exact structured evidence matches. Currently that is only neopolita/smollm-135m-gguf F16 at the pinned artifact, CPU import route, ONNX Runtime 1.29.0, and HuggingFaceTB/SmolLM-135M tokenizer revision. | | `llama-embed` | — | none (fails before config extraction) | audited-direct-loader-conditional-union | config=deferred; tensor_map=deferred; graph=deferred; runtime=deferred; quantized_import=supported | llama-embed is a canonical embedding architecture that inherits Llama's conditional tensor loader but exposes the embedding graph rather than causal logits. Mobius has no GGUF embedding task/package contract for this ID, so it must not alias ordinary llama. | | `llama4` | — | none (fails before config extraction) | exact-direct-loader-conditional-union | config=deferred; tensor_map=deferred; graph=deferred; runtime=deferred; quantized_import=supported | Llama4 GGUF is the text member of a paired multimodal package and may contain routed experts and architecture-specific cross-modal layer scheduling. The llama4 clip vision tower, token mixing, position IDs, and package ABI remain deferred; text-backbone similarity is not evidence that the complete GGUF tensor closure is owned. | | `maincoder` | — | none (fails before config extraction) | audited-direct-loader-conditional-union | config=deferred; tensor_map=deferred; graph=deferred; runtime=deferred; quantized_import=supported | Maincoder applies Q/K RMSNorm after RoPE and uses an exact tied-output QK-normalized SwiGLU closure. Existing generic QK-normalized graphs use different ordering, so a family alias would change attention. | diff --git a/src/mobius/__main__.py b/src/mobius/__main__.py index 475ade0c0..f7a3d36a6 100644 --- a/src/mobius/__main__.py +++ b/src/mobius/__main__.py @@ -10,6 +10,7 @@ import json import logging import os +import re from collections.abc import Sequence from typing import TYPE_CHECKING @@ -642,6 +643,16 @@ def _cmd_build_gguf(args: argparse.Namespace) -> None: "Error: dflash/eagle3 target-coupled drafts do not support standalone " "runtime packaging; omit --runtime to save the auxiliary graph and manifest." ) + tokenizer_repository = getattr(args, "tokenizer_repository", None) + tokenizer_revision = getattr(args, "tokenizer_revision", None) + if (tokenizer_repository is None) != (tokenizer_revision is None): + raise SystemExit( + "Error: --tokenizer-repository and --tokenizer-revision must be provided together." + ) + if runtime is None and tokenizer_repository is not None: + raise SystemExit( + "Error: pinned tokenizer materialization is only available with --runtime." + ) if runtime is not None: from mobius.integrations.gguf._arch_registry import get_arch_spec @@ -651,7 +662,6 @@ def _cmd_build_gguf(args: argparse.Namespace) -> None: ) from mobius.integrations.gguf._reader import GGUFModel from mobius.integrations.gguf._spec import Support - from mobius.integrations.gguf._tokenizer import inspect_gguf_tokenizer # Resolve and validate the exact selected source before graph construction # so a deferred tokenizer cannot leave a graph-only directory behind. @@ -664,12 +674,18 @@ def _cmd_build_gguf(args: argparse.Namespace) -> None: f"Error: GGUF runtime packaging for {architecture_spec.gguf_arch!r} is " f"{architecture_spec.runtime.value}: {architecture_spec.reason}" ) - tokenizer_verdict = inspect_gguf_tokenizer( - gguf_model.metadata, source=str(resolved_gguf_path), require_complete=True - ) - if not tokenizer_verdict.materialized: + if tokenizer_repository is None or tokenizer_revision is None: raise SystemExit( - f"Error: cannot emit a complete {runtime} package: {tokenizer_verdict.reason}" + "Error: GGUF runtime packaging requires --tokenizer-repository and an " + "immutable --tokenizer-revision." + ) + if tokenizer_repository.count("/") != 1 or not all(tokenizer_repository.split("/")): + raise SystemExit( + "Error: --tokenizer-repository must be an owner/repository Hub ID." + ) + if re.fullmatch(r"[0-9a-f]{40}", tokenizer_revision) is None: + raise SystemExit( + "Error: --tokenizer-revision must be an immutable 40-hex commit SHA." ) pkg = build_from_gguf( @@ -730,6 +746,9 @@ def _cmd_build_gguf(args: argparse.Namespace) -> None: output_dir, runtime=runtime, runtime_version=getattr(args, "runtime_version", None), + tokenizer_repository=tokenizer_repository, + tokenizer_revision=tokenizer_revision, + local_files_only=getattr(args, "local_files_only", False), external_data=args.external_data, max_shard_size_bytes=( _parse_size(args.max_shard_size) if args.max_shard_size else None @@ -1274,6 +1293,26 @@ def build_parser() -> argparse.ArgumentParser: "runtime-supported evidence record; it must equal the version validated there." ), ) + gguf_parser.add_argument( + "--tokenizer-repository", + default=None, + metavar="OWNER/REPO", + help=( + "Exact Hugging Face repository containing tokenizer assets for runtime " + "packaging. Requires --tokenizer-revision and must match runtime evidence." + ), + ) + gguf_parser.add_argument( + "--tokenizer-revision", + default=None, + metavar="COMMIT_SHA", + help="Immutable 40-hex revision for --tokenizer-repository.", + ) + gguf_parser.add_argument( + "--local-files-only", + action="store_true", + help="Use only already-cached pinned tokenizer assets; perform no Hub requests.", + ) gguf_parser.add_argument( "--static-cache", action="store_true", diff --git a/src/mobius/integrations/gguf/__init__.py b/src/mobius/integrations/gguf/__init__.py index 08ed50504..f9d2d082c 100644 --- a/src/mobius/integrations/gguf/__init__.py +++ b/src/mobius/integrations/gguf/__init__.py @@ -18,8 +18,8 @@ # Multimodal (text + companion mmproj vision/audio encoder) pkg = build_from_gguf("path/to/model.gguf", mmproj="path/to/mmproj.gguf") - # Runtime packaging is fail-closed and currently unavailable because no - # architecture has complete real-artifact runtime evidence. + # Runtime packaging is fail-closed and available only for an exact artifact, + # import route, runtime version, and pinned tokenizer evidence record. :func:`build_from_gguf` is the single entry point; passing ``mmproj`` delegates to :func:`build_gemma4_vlm_from_gguf` for the multimodal assembly. @@ -52,11 +52,19 @@ discover_gguf_shards, open_gguf_model, ) -from mobius.integrations.gguf._tokenizer import write_gguf_tokenizer_json +from mobius.integrations.gguf._tokenizer import ( + GGUFTokenizerAsset, + GGUFTokenizerSource, + materialize_gguf_tokenizer, + write_gguf_tokenizer_json, +) __all__ = [ "build_from_gguf", "build_gemma4_vlm_from_gguf", + "GGUFTokenizerAsset", + "GGUFTokenizerSource", + "materialize_gguf_tokenizer", "write_gguf_runtime_package", "write_gguf_tokenizer_json", "verify_gguf_reuse_manifest", diff --git a/src/mobius/integrations/gguf/_arch_registry.py b/src/mobius/integrations/gguf/_arch_registry.py index 38601a1b4..34950a399 100644 --- a/src/mobius/integrations/gguf/_arch_registry.py +++ b/src/mobius/integrations/gguf/_arch_registry.py @@ -679,8 +679,13 @@ tensor_map_recipe=("llama",), tensor_processor="llama", llama_qk_permute=True, - runtime=Support.DEFERRED, - reason=_RUNTIME_VALIDATION_PENDING, + runtime=Support.SUPPORTED, + runtime_evidence_ids=("smollm-135m-f16-onnxruntime-1.29.0",), + reason=( + "Runtime support is restricted to exact structured evidence matches. Currently " + "that is only neopolita/smollm-135m-gguf F16 at the pinned artifact, CPU import " + "route, ONNX Runtime 1.29.0, and HuggingFaceTB/SmolLM-135M tokenizer revision." + ), ), GGUFArchitectureSpec( gguf_arch="deci", diff --git a/src/mobius/integrations/gguf/_docs.py b/src/mobius/integrations/gguf/_docs.py index f47c0e159..3c207c96d 100644 --- a/src/mobius/integrations/gguf/_docs.py +++ b/src/mobius/integrations/gguf/_docs.py @@ -100,8 +100,8 @@ def _summary() -> str: ( f"| Tokenizer pre identifiers | {len(tokenizers)} | " f"{len({policy.canonical for policy in tokenizers.values()})} semantic groups; " - "all default to deferred and become exact-copy only with a validated embedded " - "`tokenizer.huggingface.json` |" + "all default to deferred and become materializable only from a validated embedded " + "`tokenizer.huggingface.json` or an exact pinned source in runtime evidence |" ), "", ( @@ -111,8 +111,9 @@ def _summary() -> str: "construction/execution only; runtime support additionally requires a pinned real " "artifact, independent parity, and deterministic generation or stateful semantics. " "Tokenizer `copy` delegates algorithm semantics to an embedded, " - "vocabulary-identical tokenizer JSON; it is not a reconstructed or independently " - "proven tokenizer." + "vocabulary-identical tokenizer JSON. A `pinned-source` route additionally binds " + "an immutable Hub revision, exact asset hashes, and all reconstructible GGUF " + "tokenizer semantics." ), ) ) diff --git a/src/mobius/integrations/gguf/_docs_test.py b/src/mobius/integrations/gguf/_docs_test.py index 9a4aeb2e9..517aceff0 100644 --- a/src/mobius/integrations/gguf/_docs_test.py +++ b/src/mobius/integrations/gguf/_docs_test.py @@ -48,9 +48,10 @@ def test_generated_census_counts_and_pin_are_closed() -> None: def test_runtime_support_requires_structured_evidence() -> None: - # Architecture evidence is not yet represented as structured records, so no - # architecture may claim runtime support by inheriting a permissive default. - assert all(spec.runtime is not Support.SUPPORTED for spec in iter_arch_specs()) + supported = [spec for spec in iter_arch_specs() if spec.runtime is Support.SUPPORTED] + assert [(spec.gguf_arch, spec.runtime_evidence_ids) for spec in supported] == [ + ("llama", ("smollm-135m-f16-onnxruntime-1.29.0",)) + ] pins = {pin.artifact_id for pin in MMPROJ_ARTIFACT_PINS} for spec in iter_projector_specs(): diff --git a/src/mobius/integrations/gguf/_runtime_evidence.py b/src/mobius/integrations/gguf/_runtime_evidence.py index ba5b9a39f..4b1decb0d 100644 --- a/src/mobius/integrations/gguf/_runtime_evidence.py +++ b/src/mobius/integrations/gguf/_runtime_evidence.py @@ -61,6 +61,8 @@ class GGUFRuntimeEvidence: config_revision: str tokenizer_repository: str tokenizer_revision: str + tokenizer_metadata_sha256: str + tokenizer_assets: tuple[tuple[str, int, str], ...] tensor_count: int tensor_qtypes: tuple[tuple[str, int], ...] import_route: str @@ -87,6 +89,7 @@ def __post_init__(self) -> None: self.config_revision, self.tokenizer_repository, self.tokenizer_revision, + self.tokenizer_metadata_sha256, self.import_route, self.graph_sha256, self.runtime_package_sha256, @@ -108,6 +111,7 @@ def __post_init__(self) -> None: len(value) != 64 for value in ( self.lfs_sha256, + self.tokenizer_metadata_sha256, self.graph_sha256, self.runtime_package_sha256, ) @@ -117,6 +121,7 @@ def __post_init__(self) -> None: for value in ( *revisions, self.lfs_sha256, + self.tokenizer_metadata_sha256, self.graph_sha256, self.runtime_package_sha256, ) @@ -129,6 +134,24 @@ def __post_init__(self) -> None: raise ValueError( "GGUF runtime evidence parity_kind must be full-logit or component" ) + asset_names = tuple(asset[0] for asset in self.tokenizer_assets) + if ( + not self.tokenizer_assets + or "tokenizer.json" not in asset_names + or asset_names != tuple(sorted(asset_names)) + or len(set(asset_names)) != len(asset_names) + or any( + filename != Path(filename).name + or size <= 0 + or len(sha256) != 64 + or not _is_hex(sha256) + for filename, size, sha256 in self.tokenizer_assets + ) + ): + raise ValueError( + "GGUF runtime evidence tokenizer_assets must be sorted, unique, " + "basename-only exact file identities including tokenizer.json" + ) if ( not self.graph_files or tuple(sorted(self.graph_files)) != self.graph_files @@ -145,14 +168,92 @@ def __post_init__(self) -> None: ) -# Empty by design: graph/import execution evidence does not satisfy this schema. -_RUNTIME_EVIDENCE: MappingProxyType[str, GGUFRuntimeEvidence] = MappingProxyType({}) - - def _is_hex(value: str) -> bool: return all(character in "0123456789abcdefABCDEF" for character in value) +_SMOLLM_F16_ROUTE = ( + '{"architecture":"llama","config_sha256":' + '"134f95e6a635d978737d712ed61ac8959acebdf080eafae838cf97f12c416430",' + '"execution_provider":"cpu","model_type":"llama","module_type":"llama",' + '"preserve_quantization":false,"registry_import":{"config_key_map":null,' + '"config_postprocessor":null,"llama_qk_permute":true,"offset_norm":false,' + '"required_metadata":[],"rope_interleave":false,"tensor_processor":"llama",' + '"v_head_reorder":false,"vlm_builder":null},"route_schema":1,"static_cache":false,' + '"task":{"class":"builtins.str","state":"text-generation"},' + '"tensor_map_recipe":["llama"]}' +) + +_SMOLLM_F16_ONNX_RUNTIME = GGUFRuntimeEvidence( + evidence_id="smollm-135m-f16-onnxruntime-1.29.0", + architecture="llama", + repository="neopolita/smollm-135m-gguf", + revision="22cca988936eafe92908e7558907c3964e10bba7", + filename="ggml-model-f16.gguf", + size=270_885_504, + lfs_sha256="ec8c775c16944a7e4b5251f97b3f848500dcc3e701b0d492ce9055cea42138a2", + config_repository="HuggingFaceTB/SmolLM-135M", + config_revision="1d461723eec654e65efdc40cf49301c89c0c92f4", + tokenizer_repository="HuggingFaceTB/SmolLM-135M", + tokenizer_revision="1d461723eec654e65efdc40cf49301c89c0c92f4", + tokenizer_metadata_sha256="46646ba36ecae43de6f9f649d217774b889e0fd405af92205319b882927493fc", + tokenizer_assets=( + ( + "special_tokens_map.json", + 831, + "e786b595b9a23148bf1630df78d9037a048ea671e48bfd3549a1e3c233742bb3", + ), + ( + "tokenizer.json", + 2_104_556, + "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", + ), + ( + "tokenizer_config.json", + 3_685, + "238ad6b60d48e471624ea70bc79e92f2611844d5016471fee8c167854bcb98e8", + ), + ), + tensor_count=272, + tensor_qtypes=(("F16", 211), ("F32", 61)), + import_route=_SMOLLM_F16_ROUTE, + graph_files=("model.onnx", "model.onnx.data"), + graph_sha256="3d242b09fcb5041d71e5914084cf00780867b3b0e32f669f8733369b19b6ea9b", + runtime_package_files=( + "gguf_tokenizer_manifest.json", + "inference_metadata.yaml", + "model.onnx", + "model.onnx.data", + "policies/cache_length_update.onnx", + "policies/decoder_state_initializer.onnx", + "policies/decoder_step_update.onnx", + "policies/generated_length_update.onnx", + "policies/last_token_logits.onnx", + "policies/termination.onnx", + "policies/termination_batch_initializer.onnx", + "policies/token_sampler.onnx", + "policies/token_state_update.onnx", + "policies/token_to_slot.onnx", + "special_tokens_map.json", + "tokenizer.json", + "tokenizer_config.json", + ), + runtime_package_sha256="5b6fdbdb1db7f7fb9423f2356820812712556abf783acb6bd63c572920031982", + parity_test="test_small_f16_gguf_cli_full_logit_and_generation_parity[smollm-135m-f16]", + parity_kind="full-logit", + deterministic_test=( + "test_small_f16_gguf_cli_full_logit_and_generation_parity[smollm-135m-f16]" + ), + stateful_semantics="dynamic KV cache prefill plus 20 cache-threaded decode steps", + runtime="onnx-genai", + runtime_version="1.29.0", +) + +_RUNTIME_EVIDENCE: MappingProxyType[str, GGUFRuntimeEvidence] = MappingProxyType( + {_SMOLLM_F16_ONNX_RUNTIME.evidence_id: _SMOLLM_F16_ONNX_RUNTIME} +) + + def runtime_evidence(evidence_id: str) -> GGUFRuntimeEvidence | None: """Return a structured evidence record by stable ID.""" return _RUNTIME_EVIDENCE.get(evidence_id) @@ -190,6 +291,8 @@ def matching_runtime_evidence( built_identity: GGUFArtifactIdentity, import_route: str, runtime_version: str | None, + tokenizer_repository: str, + tokenizer_revision: str, ) -> GGUFRuntimeEvidence: """Return exact evidence for the package source, route, and requested runtime.""" validate_runtime_evidence_ids(architecture, evidence_ids) @@ -220,6 +323,8 @@ def matching_runtime_evidence( and _RUNTIME_EVIDENCE[evidence_id].tensor_count == identity.tensor_count and _RUNTIME_EVIDENCE[evidence_id].tensor_qtypes == identity.tensor_qtypes and _RUNTIME_EVIDENCE[evidence_id].import_route == import_route + and _RUNTIME_EVIDENCE[evidence_id].tokenizer_repository == tokenizer_repository + and _RUNTIME_EVIDENCE[evidence_id].tokenizer_revision == tokenizer_revision ] if len(candidates) != 1: raise ValueError( diff --git a/src/mobius/integrations/gguf/_runtime_evidence_test.py b/src/mobius/integrations/gguf/_runtime_evidence_test.py index 047a958cb..a3d08d779 100644 --- a/src/mobius/integrations/gguf/_runtime_evidence_test.py +++ b/src/mobius/integrations/gguf/_runtime_evidence_test.py @@ -6,6 +6,7 @@ from __future__ import annotations import hashlib +from dataclasses import replace from types import MappingProxyType, SimpleNamespace import pytest @@ -33,6 +34,8 @@ def _record(payload: bytes) -> GGUFRuntimeEvidence: config_revision="b" * 40, tokenizer_repository="owner/tokenizer", tokenizer_revision="c" * 40, + tokenizer_metadata_sha256="f" * 64, + tokenizer_assets=(("tokenizer.json", 2, hashlib.sha256(b"{}").hexdigest()),), tensor_count=2, tensor_qtypes=(("F32", 1), ("Q4_K", 1)), import_route='{"route_schema":1}', @@ -60,6 +63,11 @@ def _model(): ) +def test_runtime_evidence_rejects_non_hex_tokenizer_metadata_digest() -> None: + with pytest.raises(ValueError, match="immutable 40-hex revisions and LFS SHA-256"): + replace(_record(b"pinned-gguf"), tokenizer_metadata_sha256="g" * 64) + + def test_matching_evidence_binds_arch_runtime_source_qtypes_and_route( tmp_path, monkeypatch ) -> None: @@ -83,6 +91,8 @@ def test_matching_evidence_binds_arch_runtime_source_qtypes_and_route( built_identity=gguf_artifact_identity(source, _model(), architecture="llama"), import_route=record.import_route, runtime_version="1.0.0", + tokenizer_repository=record.tokenizer_repository, + tokenizer_revision=record.tokenizer_revision, ) is record ) @@ -97,6 +107,21 @@ def test_matching_evidence_binds_arch_runtime_source_qtypes_and_route( built_identity=gguf_artifact_identity(source, _model(), architecture="llama"), import_route=record.import_route, runtime_version="1.0.0", + tokenizer_repository=record.tokenizer_repository, + tokenizer_revision=record.tokenizer_revision, + ) + with pytest.raises(ValueError, match="No unique GGUF runtime evidence"): + matching_runtime_evidence( + (record.evidence_id,), + architecture="llama", + runtime="onnx-genai", + source_path=source, + gguf_model=_model(), + built_identity=gguf_artifact_identity(source, _model(), architecture="llama"), + import_route=record.import_route, + runtime_version="1.0.0", + tokenizer_repository="attacker/replacement", + tokenizer_revision=record.tokenizer_revision, ) @@ -123,6 +148,8 @@ def test_matching_evidence_rejects_source_replaced_after_build(tmp_path, monkeyp built_identity=built_identity, import_route=record.import_route, runtime_version="1.0.0", + tokenizer_repository=record.tokenizer_repository, + tokenizer_revision=record.tokenizer_revision, ) diff --git a/src/mobius/integrations/gguf/_runtime_package.py b/src/mobius/integrations/gguf/_runtime_package.py index f6b698696..91755d51b 100644 --- a/src/mobius/integrations/gguf/_runtime_package.py +++ b/src/mobius/integrations/gguf/_runtime_package.py @@ -16,8 +16,10 @@ from __future__ import annotations +import ctypes import os import shutil +import sys import tempfile from pathlib import Path from typing import Any, Literal @@ -30,12 +32,61 @@ ) from mobius.integrations.gguf._spec import Support from mobius.integrations.gguf._tokenizer import ( + GGUFTokenizerAsset, + GGUFTokenizerSource, inspect_gguf_tokenizer, - write_gguf_tokenizer_json, + materialize_gguf_tokenizer, ) __all__ = ["write_gguf_runtime_package"] + +def _publish_directory_no_replace(stage: Path, destination: Path) -> None: + """Atomically publish a directory while refusing an existing destination.""" + if sys.platform == "linux": + libc = ctypes.CDLL(None, use_errno=True) + renameat2 = getattr(libc, "renameat2", None) + if renameat2 is None: + raise OSError( + "Atomic no-replace directory publication requires renameat2 on Linux" + ) + renameat2.argtypes = [ + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_uint, + ] + renameat2.restype = ctypes.c_int + result = renameat2( + -100, + os.fsencode(stage), + -100, + os.fsencode(destination), + 1, + ) + elif sys.platform == "darwin": + libc = ctypes.CDLL(None, use_errno=True) + renamex_np = libc.renamex_np + renamex_np.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_uint] + renamex_np.restype = ctypes.c_int + result = renamex_np(os.fsencode(stage), os.fsencode(destination), 0x00000004) + elif os.name == "nt": + os.rename(stage, destination) + return + else: + raise OSError( + f"Atomic no-replace directory publication is unsupported on {sys.platform!r}" + ) + if result != 0: + error_number = ctypes.get_errno() + raise OSError( + error_number, + os.strerror(error_number), + str(destination), + ) + + Runtime = Literal["onnx-genai", "ort-genai"] @@ -46,34 +97,38 @@ def write_gguf_runtime_package( *, runtime: Runtime = "onnx-genai", runtime_version: str | None = None, + tokenizer_repository: str | None = None, + tokenizer_revision: str | None = None, + local_files_only: bool = False, save_model: bool = True, **save_kwargs: Any, ) -> dict[str, str]: """Write a complete, loadable package for a runtime-evidenced GGUF model. - Emission is gated by the architecture runtime verdict. The current registry - intentionally has no runtime-supported architectures; graph-only imports remain - available. Once a qualifying structured evidence record is registered, this - function emits the graph, an exact embedded ``tokenizer.huggingface.json`` copy, - and the selected runtime's configuration contract as one staged directory. + Emission is gated by the architecture runtime verdict and an exact structured + evidence match. The function emits the graph, hash-verified tokenizer assets + from an immutable Hub revision, and the selected runtime's configuration + contract as one staged directory. Args: pkg: The :class:`~mobius.ModelPackage` returned by :func:`~mobius.integrations.gguf.build_from_gguf`. gguf_path: The source ``.gguf`` file. Runtime packaging is rejected when - it does not embed a complete, vocabulary-identical tokenizer JSON. + its tokenizer metadata does not match the selected evidence record. output_dir: Destination directory. runtime: Which runtime contract to emit. ``"onnx-genai"`` writes ``inference_metadata.yaml``; ``"ort-genai"`` writes ``genai_config.json``. runtime_version: Exact runtime version covered by the evidence record. + tokenizer_repository: Exact Hub repository holding tokenizer assets. + tokenizer_revision: Immutable 40-hex tokenizer repository revision. + local_files_only: Resolve tokenizer assets only from the local Hub cache. save_model: Must remain ``True``. Existing graph directories cannot be associated with the build-time evidence transaction safely. **save_kwargs: Forwarded to :meth:`ModelPackage.save`. Returns: - Mapping of artifact name to written path. The ``tokenizer`` key is - absent when the GGUF carries no tokenizer metadata to rebuild from. + Mapping of artifact name to written path. Raises: ValueError: If ``runtime`` is not a supported runtime name. @@ -85,6 +140,17 @@ def write_gguf_runtime_package( "save_model=False is not supported for runtime-evidenced GGUF packages because " "an existing graph cannot be bound to the build-time evidence transaction." ) + if tokenizer_repository is None or tokenizer_revision is None: + raise ValueError( + "GGUF runtime packaging requires an explicit tokenizer_repository and immutable " + "tokenizer_revision." + ) + output_dir = Path(output_dir) + if output_dir.exists(): + raise FileExistsError( + f"GGUF runtime package destination already exists: {output_dir}. " + "Refusing a non-atomic directory replacement." + ) architecture = getattr(pkg, "gguf_architecture", None) if not architecture: raise ValueError( @@ -130,7 +196,6 @@ def write_gguf_runtime_package( "sidecar contract; refusing to emit an unreachable mtp/model.onnx." ) - output_dir = Path(output_dir) source_path = Path(getattr(pkg, "gguf_source_path", gguf_path)) source_model = GGUFModel(source_path) source_architecture = get_arch_spec(source_model.architecture).gguf_arch @@ -149,6 +214,8 @@ def write_gguf_runtime_package( built_identity=built_identity, import_route=import_route, runtime_version=runtime_version, + tokenizer_repository=tokenizer_repository, + tokenizer_revision=tokenizer_revision, ) source_metadata = source_model.metadata verdict = inspect_gguf_tokenizer( @@ -156,12 +223,6 @@ def write_gguf_runtime_package( source=str(source_path), require_complete=True, ) - if not verdict.materialized: - raise ValueError( - f"Cannot emit a complete {runtime} package: {verdict.reason}. " - "The GGUF graph remains buildable, but Mobius will not claim a runnable " - "package without an exact tokenizer artifact." - ) built_verdict = getattr(pkg, "gguf_tokenizer_verdict", None) if ( built_verdict is None @@ -191,12 +252,22 @@ def write_gguf_runtime_package( f"got files={graph_identity.files}, sha256={graph_identity.sha256}." ) - tokenizer_path = write_gguf_tokenizer_json( + tokenizer_source = GGUFTokenizerSource( + repository=evidence.tokenizer_repository, + revision=evidence.tokenizer_revision, + metadata_sha256=evidence.tokenizer_metadata_sha256, + assets=tuple( + GGUFTokenizerAsset(filename, size, sha256) + for filename, size, sha256 in evidence.tokenizer_assets + ), + ) + tokenizer_path = materialize_gguf_tokenizer( source_path, stage, + source=tokenizer_source, metadata=source_metadata, - expected_metadata_sha256=built_verdict.metadata_sha256, source_identity=(f"sha256:{built_identity.sha256}/{built_identity.filename}"), + local_files_only=local_files_only, ) artifacts["tokenizer"] = tokenizer_path @@ -241,20 +312,7 @@ def write_gguf_runtime_package( f"sha256={evidence.runtime_package_sha256}; " f"got files={runtime_identity.files}, sha256={runtime_identity.sha256}." ) - backup: Path | None = None - if output_dir.exists(): - backup = output_dir.with_name(f".{output_dir.name}.backup") - if backup.exists(): - raise FileExistsError(f"Atomic package backup path already exists: {backup}") - os.replace(output_dir, backup) - try: - os.replace(stage, output_dir) - except Exception: - if backup is not None: - os.replace(backup, output_dir) - raise - if backup is not None: - shutil.rmtree(backup) + _publish_directory_no_replace(stage, output_dir) return { name: str(output_dir / Path(path).relative_to(stage)) for name, path in artifacts.items() diff --git a/src/mobius/integrations/gguf/_runtime_package_test.py b/src/mobius/integrations/gguf/_runtime_package_test.py index f115d95af..ff6b91ad9 100644 --- a/src/mobius/integrations/gguf/_runtime_package_test.py +++ b/src/mobius/integrations/gguf/_runtime_package_test.py @@ -11,9 +11,12 @@ import pytest -from mobius.integrations.gguf import write_gguf_runtime_package +from mobius.integrations.gguf import _runtime_package, write_gguf_runtime_package from mobius.integrations.gguf._spec import Support +_TOKENIZER_REPOSITORY = "owner/tokenizer" +_TOKENIZER_REVISION = "c" * 40 + class _FakePackage: def __init__(self): @@ -40,7 +43,7 @@ def _materialized(): return SimpleNamespace( materialized=True, reason="exact embedded tokenizer", - metadata_sha256="tokenizer-metadata", + metadata_sha256="f" * 64, ) @@ -56,6 +59,17 @@ def _write_config(_pkg, output, **_kwargs): return {"inference_metadata": str(path)} +def _write_runtime(pkg, source, output, **kwargs): + return write_gguf_runtime_package( + pkg, + source, + output, + tokenizer_repository=_TOKENIZER_REPOSITORY, + tokenizer_revision=_TOKENIZER_REVISION, + **kwargs, + ) + + @pytest.fixture(autouse=True) def _runtime_supported(): with ( @@ -76,6 +90,10 @@ def _runtime_supported(): graph_sha256=mock.ANY, runtime_package_files=("model.onnx",), runtime_package_sha256=mock.ANY, + tokenizer_repository=_TOKENIZER_REPOSITORY, + tokenizer_revision=_TOKENIZER_REVISION, + tokenizer_metadata_sha256="f" * 64, + tokenizer_assets=(("tokenizer.json", 2, "a" * 64),), ), ), mock.patch( @@ -87,6 +105,19 @@ def _runtime_supported(): class TestWriteGgufRuntimePackage: + def test_atomic_publication_refuses_concurrent_destination(self, tmp_path): + stage = tmp_path / "stage" + stage.mkdir() + (stage / "model.onnx").write_bytes(b"staged") + output = tmp_path / "output" + output.mkdir() + + with pytest.raises(FileExistsError): + _runtime_package._publish_directory_no_replace(stage, output) + + assert (stage / "model.onnx").read_bytes() == b"staged" + assert not list(output.iterdir()) + def test_deferred_architecture_rejects_before_source_read_or_output(self, tmp_path): pkg = _FakePackage() out = tmp_path / "out" @@ -101,7 +132,7 @@ def test_deferred_architecture_rejects_before_source_read_or_output(self, tmp_pa mock.patch("mobius.integrations.gguf._runtime_package.GGUFModel") as read_source, pytest.raises(ValueError, match=r"runtime packaging.*deferred"), ): - write_gguf_runtime_package(pkg, tmp_path / "m.gguf", out) + _write_runtime(pkg, tmp_path / "m.gguf", out) read_source.assert_not_called() assert not out.exists() @@ -118,7 +149,7 @@ def test_atomically_emits_graph_tokenizer_and_runtime_config(self, tmp_path): return_value=_materialized(), ), mock.patch( - "mobius.integrations.gguf._runtime_package.write_gguf_tokenizer_json", + "mobius.integrations.gguf._runtime_package.materialize_gguf_tokenizer", side_effect=_write_tokenizer, ), mock.patch( @@ -126,33 +157,22 @@ def test_atomically_emits_graph_tokenizer_and_runtime_config(self, tmp_path): side_effect=_write_config, ), ): - artifacts = write_gguf_runtime_package(pkg, tmp_path / "m.gguf", out) + artifacts = _write_runtime(pkg, tmp_path / "m.gguf", out) assert (out / "model.onnx").read_bytes() == b"stub" assert Path(artifacts["tokenizer"]) == out / "tokenizer.json" assert Path(artifacts["inference_metadata"]) == out / "inference_metadata.yaml" assert not list(tmp_path.glob(".out.*.tmp")) - def test_deferred_tokenizer_rejects_before_save_or_output(self, tmp_path): + def test_missing_pinned_tokenizer_source_rejects_before_save_or_output(self, tmp_path): pkg = _FakePackage() - pkg.gguf_tokenizer_verdict = SimpleNamespace(metadata_sha256="deferred") out = tmp_path / "out" with ( - mock.patch( - "mobius.integrations.gguf._runtime_package.GGUFModel", - return_value=SimpleNamespace(metadata={}, architecture="llama"), - ), - mock.patch( - "mobius.integrations.gguf._runtime_package.inspect_gguf_tokenizer", - return_value=SimpleNamespace( - materialized=False, - reason="pre is deferred", - metadata_sha256="deferred", - ), - ), - pytest.raises(ValueError, match="will not claim a runnable package"), + mock.patch("mobius.integrations.gguf._runtime_package.GGUFModel") as read_source, + pytest.raises(ValueError, match="explicit tokenizer_repository"), ): write_gguf_runtime_package(pkg, tmp_path / "m.gguf", out) + read_source.assert_not_called() assert pkg.saved_to is None assert not out.exists() @@ -174,36 +194,22 @@ def test_replaced_source_tokenizer_rejects_before_save_or_output(self, tmp_path) ), pytest.raises(ValueError, match="replaced tokenizer source"), ): - write_gguf_runtime_package(pkg, tmp_path / "m.gguf", out) + _write_runtime(pkg, tmp_path / "m.gguf", out) assert pkg.saved_to is None assert not out.exists() - def test_failed_config_write_leaves_existing_output_unchanged(self, tmp_path): + def test_existing_output_is_never_replaced(self, tmp_path): pkg = _FakePackage() out = tmp_path / "out" out.mkdir() sentinel = out / "sentinel.bin" sentinel.write_bytes(b"unchanged") with ( - mock.patch( - "mobius.integrations.gguf._runtime_package.GGUFModel", - return_value=SimpleNamespace(metadata={}, architecture="llama"), - ), - mock.patch( - "mobius.integrations.gguf._runtime_package.inspect_gguf_tokenizer", - return_value=_materialized(), - ), - mock.patch( - "mobius.integrations.gguf._runtime_package.write_gguf_tokenizer_json", - side_effect=_write_tokenizer, - ), - mock.patch( - "mobius.integrations.onnx_genai.write_onnx_genai_config", - side_effect=RuntimeError("config failed"), - ), - pytest.raises(RuntimeError, match="config failed"), + mock.patch("mobius.integrations.gguf._runtime_package.GGUFModel") as read_source, + pytest.raises(FileExistsError, match="non-atomic directory replacement"), ): - write_gguf_runtime_package(pkg, tmp_path / "m.gguf", out) + _write_runtime(pkg, tmp_path / "m.gguf", out) + read_source.assert_not_called() assert {path.name: path.read_bytes() for path in out.iterdir()} == { "sentinel.bin": b"unchanged" } @@ -212,7 +218,7 @@ def test_ort_genai_rejects_reused_gguf_weights(self, tmp_path): pkg = _FakePackage() pkg.gguf_reuse_plan = object() with pytest.raises(ValueError, match="no supported setting"): - write_gguf_runtime_package( + _write_runtime( pkg, tmp_path / "m.gguf", tmp_path / "out", @@ -224,20 +230,15 @@ def test_target_coupled_draft_runtime_package_is_rejected(self, tmp_path): pkg = _FakePackage() pkg.draft_manifest = {"architecture": "eagle3"} out = tmp_path / "out" - out.mkdir() - sentinel = out / "sentinel.bin" - sentinel.write_bytes(b"unchanged") with pytest.raises(ValueError, match="target-coupled speculative draft"): - write_gguf_runtime_package( + _write_runtime( pkg, tmp_path / "eagle3.gguf", out, ) assert pkg.saved_to is None - assert {path.name: path.read_bytes() for path in out.iterdir()} == { - "sentinel.bin": b"unchanged" - } + assert not out.exists() @pytest.mark.parametrize("runtime", ["onnx-genai", "ort-genai"]) def test_runtime_rejects_unevidenced_mtp_before_source_read(self, tmp_path, runtime): @@ -248,7 +249,7 @@ def test_runtime_rejects_unevidenced_mtp_before_source_read(self, tmp_path, runt mock.patch("mobius.integrations.gguf._runtime_package.GGUFModel") as read_source, pytest.raises(ValueError, match="runtime-evidenced GGUF MTP"), ): - write_gguf_runtime_package(pkg, tmp_path / "m.gguf", out, runtime=runtime) + _write_runtime(pkg, tmp_path / "m.gguf", out, runtime=runtime) read_source.assert_not_called() assert not out.exists() @@ -272,4 +273,4 @@ def test_target_coupled_draft_rejects_before_source_read(self, tmp_path): pkg = _FakePackage() pkg.draft_manifest = {"architecture": "eagle3"} with pytest.raises(ValueError, match="target-coupled speculative draft"): - write_gguf_runtime_package(pkg, tmp_path / "m.gguf", tmp_path / "out") + _write_runtime(pkg, tmp_path / "m.gguf", tmp_path / "out") diff --git a/src/mobius/integrations/gguf/_tokenizer.py b/src/mobius/integrations/gguf/_tokenizer.py index 30bffd930..89b06d15e 100644 --- a/src/mobius/integrations/gguf/_tokenizer.py +++ b/src/mobius/integrations/gguf/_tokenizer.py @@ -6,8 +6,11 @@ from __future__ import annotations __all__ = [ + "GGUFTokenizerAsset", + "GGUFTokenizerSource", "GGUFTokenizerVerdict", "inspect_gguf_tokenizer", + "materialize_gguf_tokenizer", "write_gguf_tokenizer_json", ] @@ -15,13 +18,18 @@ import hashlib import json import math +import os +import re +import stat +import tempfile from collections.abc import Mapping from pathlib import Path from typing import Any, Literal +from urllib.parse import urlparse from mobius.integrations.gguf._tokenizer_registry import tokenizer_pre_policies -TokenizerRoute = Literal["copy", "deferred"] +TokenizerRoute = Literal["copy", "pinned-source", "deferred"] _BPE_MODELS = frozenset({"gpt2", "hybriddna", "whitespace", "gemma4"}) _KNOWN_MODELS = frozenset( @@ -56,6 +64,38 @@ "tokenizer.ggml.normalizer.lowercase", "tokenizer.ggml.normalizer.strip_accents", ) +_TOKENIZER_ASSET_NAMES = frozenset( + { + "added_tokens.json", + "chat_template.jinja", + "special_tokens_map.json", + "tokenizer.json", + "tokenizer_config.json", + } +) +_MAX_TOKENIZER_ASSET_BYTES = 64 * 1024 * 1024 +_SMOLLM_PIPELINE = { + "normalizer": None, + "pre_tokenizer": { + "type": "Sequence", + "pretokenizers": [ + {"type": "Digits", "individual_digits": True}, + { + "type": "ByteLevel", + "add_prefix_space": False, + "trim_offsets": True, + "use_regex": True, + }, + ], + }, + "post_processor": None, + "decoder": { + "type": "ByteLevel", + "add_prefix_space": True, + "trim_offsets": True, + "use_regex": True, + }, +} # Audited against the pinned C++ loader. ``tokenizer.huggingface.json`` and # ``tokenizer.chat_templates`` are converter/extension fields; llama.cpp does @@ -96,7 +136,55 @@ class GGUFTokenizerVerdict: @property def materialized(self) -> bool: - return self.route == "copy" + return self.route in {"copy", "pinned-source"} + + +@dataclasses.dataclass(frozen=True, slots=True) +class GGUFTokenizerAsset: + """Expected identity of one exact tokenizer source file.""" + + filename: str + size: int + sha256: str + + def __post_init__(self) -> None: + if self.filename not in _TOKENIZER_ASSET_NAMES: + raise ValueError(f"Unsupported tokenizer asset filename: {self.filename!r}") + if self.size <= 0 or self.size > _MAX_TOKENIZER_ASSET_BYTES: + raise ValueError( + f"Invalid tokenizer asset size for {self.filename!r}: {self.size}" + ) + if re.fullmatch(r"[0-9a-f]{64}", self.sha256) is None: + raise ValueError(f"Tokenizer asset {self.filename!r} requires a lowercase SHA-256") + + +@dataclasses.dataclass(frozen=True, slots=True) +class GGUFTokenizerSource: + """Immutable Hub source and exact file identities for tokenizer materialization.""" + + repository: str + revision: str + assets: tuple[GGUFTokenizerAsset, ...] + metadata_sha256: str + + def __post_init__(self) -> None: + if self.repository.count("/") != 1 or not all(self.repository.split("/")): + raise ValueError("Tokenizer source repository must be an owner/repository Hub ID") + if re.fullmatch(r"[0-9a-f]{40}", self.revision) is None: + raise ValueError( + "Tokenizer source revision must be an immutable 40-hex commit SHA" + ) + if re.fullmatch(r"[0-9a-f]{64}", self.metadata_sha256) is None: + raise ValueError( + "Tokenizer source requires the exact GGUF tokenizer metadata SHA-256" + ) + names = tuple(asset.filename for asset in self.assets) + if "tokenizer.json" not in names: + raise ValueError("Tokenizer source must include tokenizer.json") + if len(set(names)) != len(names): + raise ValueError("Tokenizer source contains duplicate asset filenames") + if names != tuple(sorted(names)): + raise ValueError("Tokenizer source assets must be sorted by filename") def _require_list(metadata: Mapping[str, Any], key: str) -> list[Any] | None: @@ -366,6 +454,7 @@ def inspect_gguf_tokenizer( policy.canonical if policy else None, f"{detail}; exact ORT tokenizer materialization is unavailable", len(tokens), + metadata_sha256=_tokenizer_metadata_sha256(metadata), ) @@ -476,3 +565,456 @@ def write_gguf_tokenizer_json( json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8" ) return str(tokenizer_path) + + +def _read_regular_file(path: Path, *, expected: GGUFTokenizerAsset) -> bytes: + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + try: + before = os.fstat(descriptor) + if not stat.S_ISREG(before.st_mode) or path.is_symlink(): + raise ValueError(f"Tokenizer asset must be a non-symlink regular file: {path}") + chunks: list[bytes] = [] + digest = hashlib.sha256() + size = 0 + while chunk := os.read(descriptor, 1024 * 1024): + size += len(chunk) + if size > _MAX_TOKENIZER_ASSET_BYTES: + raise ValueError(f"Tokenizer asset exceeds size limit: {expected.filename}") + chunks.append(chunk) + digest.update(chunk) + after = os.fstat(descriptor) + current = path.stat() + + def identity(value: os.stat_result) -> tuple[int, int, int, int]: + return (value.st_dev, value.st_ino, value.st_size, value.st_mtime_ns) + + if identity(before) != identity(after) or identity(after) != identity(current): + raise ValueError(f"Tokenizer asset changed while it was being read: {path}") + if size != expected.size or digest.hexdigest() != expected.sha256: + raise ValueError( + f"Tokenizer asset identity mismatch for {expected.filename}: " + f"expected size={expected.size}, sha256={expected.sha256}; " + f"got size={size}, sha256={digest.hexdigest()}" + ) + return b"".join(chunks) + finally: + os.close(descriptor) + + +def _validate_asset_payload(payload: bytes, *, expected: GGUFTokenizerAsset) -> bytes: + digest = hashlib.sha256(payload).hexdigest() + if len(payload) != expected.size or digest != expected.sha256: + raise ValueError( + f"Tokenizer asset identity mismatch for {expected.filename}: " + f"expected size={expected.size}, sha256={expected.sha256}; " + f"got size={len(payload)}, sha256={digest}" + ) + return payload + + +def _download_tokenizer_assets( + source: GGUFTokenizerSource, + *, + local_files_only: bool, +) -> dict[str, bytes]: + from huggingface_hub import ( + get_hf_file_metadata, + get_session, + hf_hub_download, + hf_hub_url, + ) + from huggingface_hub.utils import build_hf_headers + + payloads: dict[str, bytes] = {} + for asset in source.assets: + if local_files_only: + from huggingface_hub.constants import HF_HUB_CACHE + + path = Path( + hf_hub_download( + repo_id=source.repository, + revision=source.revision, + filename=asset.filename, + local_files_only=True, + ) + ) + if path.is_symlink(): + try: + path.absolute().relative_to(Path(HF_HUB_CACHE).absolute()) + except ValueError as error: + raise ValueError( + f"Cached tokenizer asset is an untrusted symlink: {path}" + ) from error + path = path.resolve(strict=True) + else: + url = hf_hub_url(source.repository, asset.filename, revision=source.revision) + metadata = get_hf_file_metadata(url) + if metadata.commit_hash != source.revision: + raise ValueError( + f"Hub resolved {source.repository}:{asset.filename} to " + f"{metadata.commit_hash!r}, not pinned revision {source.revision!r}" + ) + location = urlparse(metadata.location) + if ( + location.scheme != "https" + or not location.netloc + or location.username is not None + or location.password is not None + ): + raise ValueError( + f"Hub returned an unsafe tokenizer asset location: {metadata.location!r}" + ) + headers = build_hf_headers() + if urlparse(url).netloc != location.netloc: + for name in tuple(headers): + if name.lower() == "authorization": + headers.pop(name) + session = get_session() + + def read_response( + response: Any, + *, + location: str = metadata.location, + expected_asset: GGUFTokenizerAsset = asset, + ) -> bytes: + if 300 <= response.status_code < 400: + raise ValueError( + "Tokenizer asset endpoint redirected after authorization policy " + f"was selected: {location}" + ) + response.raise_for_status() + chunks: list[bytes] = [] + size = 0 + iterator = ( + response.iter_bytes() + if hasattr(response, "iter_bytes") + else response.iter_content(chunk_size=1024 * 1024) + ) + for chunk in iterator: + size += len(chunk) + if size > expected_asset.size or size > _MAX_TOKENIZER_ASSET_BYTES: + raise ValueError( + f"Tokenizer asset exceeded evidenced size: {expected_asset.filename}" + ) + chunks.append(chunk) + return b"".join(chunks) + + stream = getattr(session, "stream", None) + if callable(stream): + with stream( + "GET", + metadata.location, + headers=headers, + follow_redirects=False, + ) as response: + payload = read_response(response) + else: + with session.get( + metadata.location, + headers=headers, + allow_redirects=False, + stream=True, + ) as response: + payload = read_response(response) + payloads[asset.filename] = _validate_asset_payload(payload, expected=asset) + continue + payloads[asset.filename] = _read_regular_file(path, expected=asset) + return payloads + + +def _json_object(payload: bytes, *, filename: str) -> dict[str, Any]: + try: + value = json.loads(payload) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise ValueError(f"{filename} is not valid UTF-8 JSON") from error + if not isinstance(value, dict): + raise TypeError(f"{filename} must contain a JSON object") + return value + + +def _merge_pairs(value: Any) -> list[tuple[str, str]]: + if not isinstance(value, list): + raise TypeError("tokenizer.json model.merges must be an array") + pairs: list[tuple[str, str]] = [] + for index, merge in enumerate(value): + if isinstance(merge, str): + separator = merge.find(" ", 1) + if separator < 0: + raise ValueError(f"tokenizer.json model.merges[{index}] is malformed") + pair = (merge[:separator], merge[separator + 1 :]) + elif ( + isinstance(merge, list) + and len(merge) == 2 + and all(isinstance(token, str) for token in merge) + ): + pair = (merge[0], merge[1]) + else: + raise ValueError(f"tokenizer.json model.merges[{index}] is malformed") + if not all(pair): + raise ValueError(f"tokenizer.json model.merges[{index}] is malformed") + pairs.append(pair) + if len(set(pairs)) != len(pairs): + raise ValueError("tokenizer.json model.merges contains duplicate pairs") + return pairs + + +def _special_token_content(value: Any, *, key: str) -> str: + if isinstance(value, str): + return value + if isinstance(value, Mapping) and isinstance(value.get("content"), str): + return value["content"] + raise ValueError(f"tokenizer_config.json {key} must be a string or token object") + + +def _validate_pinned_tokenizer( + metadata: Mapping[str, Any], + payloads: Mapping[str, bytes], +) -> str: + raw_tokenizer = payloads["tokenizer.json"] + tokenizer_json = _json_object(raw_tokenizer, filename="tokenizer.json") + config = _json_object( + payloads.get("tokenizer_config.json", b"{}"), filename="tokenizer_config.json" + ) + special_map = _json_object( + payloads.get("special_tokens_map.json", b"{}"), + filename="special_tokens_map.json", + ) + try: + from tokenizers import Tokenizer + + tokenizer = Tokenizer.from_str(raw_tokenizer.decode("utf-8")) + except Exception as error: + raise ValueError("tokenizer.json is not a loadable tokenizers tokenizer") from error + + expected_tokens = metadata["tokenizer.ggml.tokens"] + actual_tokens = [ + tokenizer.id_to_token(index) + for index in range(tokenizer.get_vocab_size(with_added_tokens=True)) + ] + if actual_tokens != expected_tokens: + mismatch = next( + ( + index + for index, (actual, expected) in enumerate(zip(actual_tokens, expected_tokens)) + if actual != expected + ), + min(len(actual_tokens), len(expected_tokens)), + ) + raise ValueError( + f"Pinned tokenizer vocabulary differs from GGUF at token id {mismatch}" + ) + + model = tokenizer_json.get("model") + if not isinstance(model, Mapping): + raise TypeError("tokenizer.json must contain a model object") + if _merge_pairs(model.get("merges")) != _merge_pairs( + metadata.get("tokenizer.ggml.merges") + ): + raise ValueError( + "Pinned tokenizer merge order differs from GGUF tokenizer.ggml.merges" + ) + pre = metadata.get("tokenizer.ggml.pre") + expected_pipeline = _SMOLLM_PIPELINE if pre == "smollm" else None + if expected_pipeline is None: + raise ValueError( + f"Pinned tokenizer pipeline validation is not implemented for GGUF pre {pre!r}" + ) + actual_pipeline = { + name: tokenizer_json.get(name) + for name in ("normalizer", "pre_tokenizer", "post_processor", "decoder") + } + if actual_pipeline != expected_pipeline: + raise ValueError(f"Pinned tokenizer pipeline differs from GGUF pre {pre!r}") + token_types = metadata.get("tokenizer.ggml.token_type") + if token_types is not None: + unsupported_types = sorted(set(token_types) - {1, 2, 3}) + if unsupported_types: + raise ValueError( + f"Pinned tokenizer identity cannot prove GGUF token types {unsupported_types}" + ) + source_special_ids = { + token["id"] + for token in tokenizer_json.get("added_tokens", ()) + if isinstance(token, Mapping) and token.get("special") is True + } + expected_special_ids = { + index for index, token_type in enumerate(token_types) if token_type in {2, 3} + } + if source_special_ids != expected_special_ids: + raise ValueError("Pinned tokenizer special-token inventory differs from GGUF") + + special_names = { + "bos_token_id": "bos_token", + "eos_token_id": "eos_token", + "unknown_token_id": "unk_token", + "padding_token_id": "pad_token", + "seperator_token_id": "sep_token", + "cls_token_id": "cls_token", + "mask_token_id": "mask_token", + } + for gguf_suffix, config_name in special_names.items(): + expected_id = metadata.get(f"tokenizer.ggml.{gguf_suffix}") + if expected_id is None: + continue + raw_value = config.get(config_name, special_map.get(config_name)) + if raw_value is None: + raise ValueError(f"Pinned tokenizer omits GGUF special token {config_name}") + token = _special_token_content(raw_value, key=config_name) + if tokenizer.token_to_id(token) != expected_id: + raise ValueError(f"Pinned tokenizer {config_name} id differs from GGUF") + + for gguf_name, config_name in ( + ("add_bos_token", "add_bos_token"), + ("add_eos_token", "add_eos_token"), + ("add_sep_token", "add_sep_token"), + ): + expected = metadata.get(f"tokenizer.ggml.{gguf_name}") + actual = config.get(config_name) + if expected is None: + continue + if actual is None: + if expected is not False: + raise ValueError(f"Pinned tokenizer cannot prove GGUF {config_name}") + elif actual is not expected: + raise ValueError(f"Pinned tokenizer {config_name} differs from GGUF") + + expected_prefix = metadata.get("tokenizer.ggml.add_space_prefix") + if expected_prefix is not None: + pre_tokenizer = tokenizer_json.get("pre_tokenizer") + queue = [pre_tokenizer] + byte_level: list[Mapping[str, Any]] = [] + while queue: + current = queue.pop() + if not isinstance(current, Mapping): + continue + if current.get("type") == "ByteLevel": + byte_level.append(current) + children = current.get("pretokenizers") + if isinstance(children, list): + queue.extend(children) + if ( + len(byte_level) != 1 + or byte_level[0].get("add_prefix_space") is not expected_prefix + ): + raise ValueError("Pinned tokenizer add_prefix_space differs from GGUF") + + suppress = metadata.get("tokenizer.ggml.suppress_tokens") + if suppress is not None and config.get("suppress_tokens") != suppress: + raise ValueError("Pinned tokenizer suppress_tokens differs from GGUF") + validated_metadata = { + "tokenizer.ggml.model", + "tokenizer.ggml.pre", + "tokenizer.ggml.tokens", + "tokenizer.ggml.merges", + "tokenizer.ggml.token_type", + "tokenizer.ggml.bos_token_id", + "tokenizer.ggml.eos_token_id", + "tokenizer.ggml.unknown_token_id", + "tokenizer.ggml.padding_token_id", + "tokenizer.ggml.seperator_token_id", + "tokenizer.ggml.cls_token_id", + "tokenizer.ggml.mask_token_id", + "tokenizer.ggml.add_bos_token", + "tokenizer.ggml.add_eos_token", + "tokenizer.ggml.add_sep_token", + "tokenizer.ggml.add_space_prefix", + "tokenizer.ggml.suppress_tokens", + "tokenizer.chat_template", + "tokenizer.chat_templates", + } + validated_metadata.update( + key for key in metadata if key.startswith("tokenizer.chat_template.") + ) + unsupported = sorted( + key + for key in metadata + if key.startswith("tokenizer.") + and key not in validated_metadata + and key != "tokenizer.huggingface.json" + ) + if unsupported: + raise ValueError( + f"Pinned tokenizer identity cannot prove GGUF tokenizer fields {unsupported}" + ) + + templates = _validate_chat_templates(metadata) + source_template = config.get("chat_template") + if "chat_template.jinja" in payloads: + file_template = payloads["chat_template.jinja"].decode("utf-8") + if source_template is not None and source_template != file_template: + raise ValueError("Pinned tokenizer chat template assets contradict each other") + source_template = file_template + if templates: + expected_template: Any = ( + templates["default"] + if set(templates) == {"default"} + else dict(sorted(templates.items())) + ) + if source_template != expected_template: + raise ValueError("Pinned tokenizer chat template differs from GGUF") + + return hashlib.sha256(raw_tokenizer).hexdigest() + + +def materialize_gguf_tokenizer( + gguf_path: str | Path, + output_dir: str | Path, + *, + source: GGUFTokenizerSource, + metadata: Mapping[str, Any] | None = None, + source_identity: str | None = None, + local_files_only: bool = False, +) -> str: + """Materialize exact pinned tokenizer assets after fail-closed semantic validation.""" + from mobius.integrations.gguf._reader import GGUFModel + + gguf_path = Path(gguf_path) + if metadata is None: + metadata = GGUFModel(gguf_path).metadata + verdict = inspect_gguf_tokenizer(metadata, source=str(gguf_path), require_complete=True) + if verdict.metadata_sha256 != source.metadata_sha256: + raise ValueError("Pinned tokenizer evidence does not match GGUF tokenizer metadata") + payloads = _download_tokenizer_assets(source, local_files_only=local_files_only) + tokenizer_sha256 = _validate_pinned_tokenizer(metadata, payloads) + + output = Path(output_dir) + output.mkdir(parents=True, exist_ok=True) + manifest = { + "format_version": 1, + "source": source_identity or str(gguf_path.resolve()), + "route": "pinned-source", + "model": verdict.model, + "pre": verdict.pre, + "canonical_pre": verdict.canonical_pre, + "token_count": verdict.token_count, + "tokenizer_sha256": tokenizer_sha256, + "metadata_sha256": verdict.metadata_sha256, + "tokenizer_repository": source.repository, + "tokenizer_revision": source.revision, + "assets": [dataclasses.asdict(asset) for asset in source.assets], + "pipeline_semantics": "exact_pinned_tokenizer_assets", + "ort_genai_compatible": "validated", + } + writes = dict(payloads) + writes["gguf_tokenizer_manifest.json"] = ( + json.dumps(manifest, indent=2, sort_keys=True) + "\n" + ).encode() + temporary: list[tuple[Path, Path]] = [] + try: + for filename, payload in writes.items(): + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{filename}.", suffix=".tmp", dir=output + ) + temporary_path = Path(temporary_name) + with os.fdopen(descriptor, "wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + temporary.append((temporary_path, output / filename)) + for temporary_path, destination in temporary: + os.replace(temporary_path, destination) + return str(output / "tokenizer.json") + finally: + for temporary_path, _ in temporary: + temporary_path.unlink(missing_ok=True) diff --git a/src/mobius/integrations/gguf/_tokenizer_test.py b/src/mobius/integrations/gguf/_tokenizer_test.py index 14074d62c..404c426a3 100644 --- a/src/mobius/integrations/gguf/_tokenizer_test.py +++ b/src/mobius/integrations/gguf/_tokenizer_test.py @@ -5,12 +5,22 @@ from __future__ import annotations +import hashlib import json +import os from pathlib import Path +from types import SimpleNamespace +from unittest import mock import pytest -from mobius.integrations.gguf._tokenizer import inspect_gguf_tokenizer +from mobius.integrations.gguf import _tokenizer +from mobius.integrations.gguf._tokenizer import ( + GGUFTokenizerAsset, + GGUFTokenizerSource, + inspect_gguf_tokenizer, + materialize_gguf_tokenizer, +) from mobius.integrations.gguf._tokenizer_registry import tokenizer_pre_policies _PINNED_PRE_IDENTIFIERS = ( @@ -324,3 +334,296 @@ def test_write_exact_tokenizer_assets_removes_stale_default_template(tmp_path: P write_gguf_tokenizer_json(source, output) assert not stale.exists() + + +def _pinned_payloads(metadata: dict) -> dict[str, bytes]: + tokenizer = json.loads(_tokenizer_json(metadata["tokenizer.ggml.tokens"])) + tokenizer["model"]["merges"] = metadata["tokenizer.ggml.merges"] + tokenizer["normalizer"] = None + tokenizer["pre_tokenizer"] = { + "type": "Sequence", + "pretokenizers": [ + {"type": "Digits", "individual_digits": True}, + { + "type": "ByteLevel", + "add_prefix_space": False, + "trim_offsets": True, + "use_regex": True, + }, + ], + } + tokenizer["post_processor"] = None + tokenizer["decoder"] = { + "type": "ByteLevel", + "add_prefix_space": True, + "trim_offsets": True, + "use_regex": True, + } + tokenizer["added_tokens"] = [ + { + "id": index, + "content": token, + "single_word": False, + "lstrip": False, + "rstrip": False, + "normalized": False, + "special": True, + } + for index, (token, token_type) in enumerate( + zip( + metadata["tokenizer.ggml.tokens"], + metadata["tokenizer.ggml.token_type"], + strict=True, + ) + ) + if token_type in {2, 3} + ] + config = { + "bos_token": metadata["tokenizer.ggml.tokens"][ + metadata["tokenizer.ggml.bos_token_id"] + ], + "eos_token": metadata["tokenizer.ggml.tokens"][ + metadata["tokenizer.ggml.eos_token_id"] + ], + "unk_token": metadata["tokenizer.ggml.tokens"][ + metadata["tokenizer.ggml.unknown_token_id"] + ], + "pad_token": metadata["tokenizer.ggml.tokens"][ + metadata["tokenizer.ggml.padding_token_id"] + ], + "add_bos_token": metadata["tokenizer.ggml.add_bos_token"], + "add_eos_token": metadata["tokenizer.ggml.add_eos_token"], + } + return { + "special_tokens_map.json": json.dumps( + {name: value for name, value in config.items() if name.endswith("_token")} + ).encode(), + "tokenizer.json": json.dumps(tokenizer).encode(), + "tokenizer_config.json": json.dumps(config).encode(), + } + + +def _pinned_source(metadata: dict, payloads: dict[str, bytes]) -> GGUFTokenizerSource: + verdict = inspect_gguf_tokenizer(metadata, require_complete=True) + return GGUFTokenizerSource( + repository="owner/tokenizer", + revision="a" * 40, + metadata_sha256=str(verdict.metadata_sha256), + assets=tuple( + GGUFTokenizerAsset(name, len(payload), hashlib.sha256(payload).hexdigest()) + for name, payload in sorted(payloads.items()) + ), + ) + + +def test_pinned_source_rejects_mutable_revision_and_duplicate_assets() -> None: + asset = GGUFTokenizerAsset("tokenizer.json", 2, hashlib.sha256(b"{}").hexdigest()) + with pytest.raises(ValueError, match="immutable 40-hex"): + GGUFTokenizerSource("owner/tokenizer", "main", (asset,), "a" * 64) + for repository in ("owner/", "/tokenizer"): + with pytest.raises(ValueError, match="owner/repository"): + GGUFTokenizerSource(repository, "a" * 40, (asset,), "a" * 64) + with pytest.raises(ValueError, match="duplicate asset"): + GGUFTokenizerSource("owner/tokenizer", "a" * 40, (asset, asset), "a" * 64) + + +def test_pinned_source_missing_tokenizer_json_rejects() -> None: + payload = b"{}" + asset = GGUFTokenizerAsset( + "tokenizer_config.json", len(payload), hashlib.sha256(payload).hexdigest() + ) + with pytest.raises(ValueError, match=r"must include tokenizer\.json"): + GGUFTokenizerSource("owner/tokenizer", "a" * 40, (asset,), "a" * 64) + + +def test_missing_pinned_hub_asset_leaves_no_output(tmp_path: Path, monkeypatch) -> None: + metadata = _metadata(pre="smollm") + payloads = _pinned_payloads(metadata) + source = _pinned_source(metadata, payloads) + monkeypatch.setattr( + "huggingface_hub.hf_hub_download", + mock.Mock(side_effect=FileNotFoundError("missing tokenizer asset")), + ) + output = tmp_path / "output" + + with pytest.raises(FileNotFoundError, match="missing tokenizer asset"): + materialize_gguf_tokenizer( + tmp_path / "model.gguf", + output, + source=source, + metadata=metadata, + local_files_only=True, + ) + + assert not output.exists() + + +def test_tokenizer_evidence_metadata_mismatch_rejects_before_download( + tmp_path: Path, monkeypatch +) -> None: + metadata = _metadata(pre="smollm") + payloads = _pinned_payloads(metadata) + source = _pinned_source(metadata, payloads) + source = GGUFTokenizerSource( + source.repository, + source.revision, + source.assets, + "b" * 64, + ) + download = mock.Mock() + monkeypatch.setattr(_tokenizer, "_download_tokenizer_assets", download) + + with pytest.raises(ValueError, match="does not match GGUF tokenizer metadata"): + materialize_gguf_tokenizer( + tmp_path / "model.gguf", + tmp_path / "output", + source=source, + metadata=metadata, + ) + + download.assert_not_called() + + +def test_semantic_mismatch_leaves_no_partial_output(tmp_path: Path, monkeypatch) -> None: + metadata = _metadata(pre="smollm") + payloads = _pinned_payloads(metadata) + config = json.loads(payloads["tokenizer_config.json"]) + config["bos_token"] = "" + payloads["tokenizer_config.json"] = json.dumps(config).encode() + source = _pinned_source(metadata, payloads) + monkeypatch.setattr(_tokenizer, "_download_tokenizer_assets", lambda *_a, **_k: payloads) + output = tmp_path / "output" + + with pytest.raises(ValueError, match="bos_token id differs"): + materialize_gguf_tokenizer( + tmp_path / "model.gguf", + output, + source=source, + metadata=metadata, + ) + + assert not output.exists() + + +def test_pipeline_mismatch_leaves_no_partial_output(tmp_path: Path, monkeypatch) -> None: + metadata = _metadata(pre="smollm") + payloads = _pinned_payloads(metadata) + tokenizer = json.loads(payloads["tokenizer.json"]) + tokenizer["pre_tokenizer"]["pretokenizers"].reverse() + payloads["tokenizer.json"] = json.dumps(tokenizer).encode() + source = _pinned_source(metadata, payloads) + monkeypatch.setattr(_tokenizer, "_download_tokenizer_assets", lambda *_a, **_k: payloads) + output = tmp_path / "output" + + with pytest.raises(ValueError, match="pipeline differs"): + materialize_gguf_tokenizer( + tmp_path / "model.gguf", + output, + source=source, + metadata=metadata, + ) + + assert not output.exists() + + +def test_post_processor_cannot_hide_matching_special_token_flags( + tmp_path: Path, monkeypatch +) -> None: + metadata = _metadata(pre="smollm") + payloads = _pinned_payloads(metadata) + tokenizer = json.loads(payloads["tokenizer.json"]) + tokenizer["post_processor"] = { + "type": "TemplateProcessing", + "single": [{"SpecialToken": {"id": "", "type_id": 0}}], + "pair": [{"Sequence": {"id": "A", "type_id": 0}}], + "special_tokens": {"": {"id": "", "ids": [2], "tokens": [""]}}, + } + payloads["tokenizer.json"] = json.dumps(tokenizer).encode() + source = _pinned_source(metadata, payloads) + monkeypatch.setattr(_tokenizer, "_download_tokenizer_assets", lambda *_a, **_k: payloads) + + with pytest.raises(ValueError, match="pipeline differs"): + materialize_gguf_tokenizer( + tmp_path / "model.gguf", + tmp_path / "output", + source=source, + metadata=metadata, + ) + + +def test_cross_host_asset_request_strips_auth_and_rejects_redirect(monkeypatch) -> None: + payload = b"{}" + source = GGUFTokenizerSource( + "owner/tokenizer", + "a" * 40, + ( + GGUFTokenizerAsset( + "tokenizer.json", len(payload), hashlib.sha256(payload).hexdigest() + ), + ), + "b" * 64, + ) + response = SimpleNamespace(status_code=302) + response.raise_for_status = lambda: None + seen_headers: dict[str, str] = {} + + class _Stream: + def __enter__(self): + return response + + def __exit__(self, *_args): + return None + + class _Session: + def stream(self, _method, _url, *, headers, follow_redirects): + assert follow_redirects is False + seen_headers.update(headers) + return _Stream() + + monkeypatch.setattr("huggingface_hub.hf_hub_url", lambda *_a, **_k: "https://hub/a") + monkeypatch.setattr( + "huggingface_hub.get_hf_file_metadata", + lambda _url: SimpleNamespace( + commit_hash="a" * 40, + location="https://cdn/tokenizer.json", + ), + ) + monkeypatch.setattr("huggingface_hub.get_session", lambda: _Session()) + monkeypatch.setattr( + "huggingface_hub.utils.build_hf_headers", + lambda: {"Authorization": "Bearer secret", "user-agent": "test"}, + ) + + with pytest.raises(ValueError, match="redirected after authorization policy"): + _tokenizer._download_tokenizer_assets(source, local_files_only=False) + + assert not {name for name in seen_headers if name.lower() == "authorization"} + + +def test_local_asset_replacement_during_read_rejects(tmp_path: Path) -> None: + path = tmp_path / "tokenizer.json" + payload = b"{}" + path.write_bytes(payload) + expected = GGUFTokenizerAsset( + "tokenizer.json", len(payload), hashlib.sha256(payload).hexdigest() + ) + first = path.stat() + changed = os.stat_result( + ( + first.st_mode, + first.st_ino + 1, + first.st_dev, + first.st_nlink, + first.st_uid, + first.st_gid, + first.st_size, + first.st_atime, + first.st_mtime, + first.st_ctime, + ) + ) + with ( + mock.patch.object(_tokenizer.os, "fstat", side_effect=[first, changed]), + pytest.raises(ValueError, match="changed while it was being read"), + ): + _tokenizer._read_regular_file(path, expected=expected) diff --git a/testdata/cases/causal-lm/smollm-135m-gguf-f16.yaml b/testdata/cases/causal-lm/smollm-135m-gguf-f16.yaml index b7316b401..0dba9a6be 100644 --- a/testdata/cases/causal-lm/smollm-135m-gguf-f16.yaml +++ b/testdata/cases/causal-lm/smollm-135m-gguf-f16.yaml @@ -18,6 +18,21 @@ gguf: config_sha256: "134f95e6a635d978737d712ed61ac8959acebdf080eafae838cf97f12c416430" preserve_quantization: false keep_quantized: true + tokenizer: + repository: "HuggingFaceTB/SmolLM-135M" + revision: "1d461723eec654e65efdc40cf49301c89c0c92f4" + metadata_sha256: "46646ba36ecae43de6f9f649d217774b889e0fd405af92205319b882927493fc" + identity_status: "exact" + assets: + - filename: "special_tokens_map.json" + size: 831 + sha256: "e786b595b9a23148bf1630df78d9037a048ea671e48bfd3549a1e3c233742bb3" + - filename: "tokenizer.json" + size: 2104556 + sha256: "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c" + - filename: "tokenizer_config.json" + size: 3685 + sha256: "238ad6b60d48e471624ea70bc79e92f2611844d5016471fee8c167854bcb98e8" inputs: prompts: diff --git a/testdata/cases/causal-lm/smollm2-135m-gguf-f16.yaml b/testdata/cases/causal-lm/smollm2-135m-gguf-f16.yaml index 8e374b128..5aa195a31 100644 --- a/testdata/cases/causal-lm/smollm2-135m-gguf-f16.yaml +++ b/testdata/cases/causal-lm/smollm2-135m-gguf-f16.yaml @@ -18,6 +18,22 @@ gguf: config_sha256: "c62123baf4e95656cdc9f5b798c14319bbaafec594526c462b10555f561969f9" preserve_quantization: false keep_quantized: true + tokenizer: + repository: "HuggingFaceTB/SmolLM2-135M-Instruct" + revision: "12fd25f77366fa6b3b4b768ec3050bf629380bac" + metadata_sha256: "cb0b637d59effdc3ab02f063039e597157fa4996663848cc2178510af5880ace" + identity_status: "rejected" + rejection_reason: "GGUF padding_token_id=0 conflicts with the pinned tokenizer pad_token_id=2." + assets: + - filename: "special_tokens_map.json" + size: 655 + sha256: "2b7379f3ae813529281a5c602bc5a11c1d4e0a99107aaa597fe936c1e813ca52" + - filename: "tokenizer.json" + size: 2104556 + sha256: "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c" + - filename: "tokenizer_config.json" + size: 3764 + sha256: "4ec77d44f62efeb38d7e044a1db318f6a939438425312dfa333b8382dbad98df" inputs: prompts: diff --git a/testdata/cases/schema.json b/testdata/cases/schema.json index f648e35bc..53c259096 100644 --- a/testdata/cases/schema.json +++ b/testdata/cases/schema.json @@ -88,7 +88,8 @@ "tensor_qtypes", "execution_provider", "config_sha256", - "preserve_quantization" + "preserve_quantization", + "tokenizer" ], "properties": { "repository": { @@ -153,6 +154,110 @@ "type": "boolean", "default": true, "description": "Whether to preserve supported GGUF quantization." + }, + "tokenizer": { + "type": "object", + "additionalProperties": false, + "required": [ + "repository", + "revision", + "metadata_sha256", + "assets", + "identity_status" + ], + "allOf": [ + { + "if": { + "properties": { + "identity_status": { + "const": "rejected" + } + } + }, + "then": { + "required": [ + "rejection_reason" + ] + }, + "else": { + "not": { + "required": [ + "rejection_reason" + ] + } + } + } + ], + "properties": { + "repository": { + "type": "string", + "pattern": "^[^/]+/[^/]+$" + }, + "revision": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "metadata_sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "assets": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "contains": { + "type": "object", + "properties": { + "filename": { + "const": "tokenizer.json" + } + }, + "required": [ + "filename" + ] + }, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "filename", + "size", + "sha256" + ], + "properties": { + "filename": { + "type": "string", + "enum": [ + "added_tokens.json", + "chat_template.jinja", + "special_tokens_map.json", + "tokenizer.json", + "tokenizer_config.json" + ] + }, + "size": { + "type": "integer", + "minimum": 1 + }, + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + } + }, + "identity_status": { + "type": "string", + "enum": [ + "exact", + "rejected" + ] + }, + "rejection_reason": { + "type": "string", + "minLength": 1 + } + } } }, "description": "Pinned GGUF import source. The top-level model_id remains the independent HuggingFace reference." diff --git a/testdata/golden/causal-lm/smollm-135m-gguf-f16.json b/testdata/golden/causal-lm/smollm-135m-gguf-f16.json index 295691691..3e3faf807 100644 --- a/testdata/golden/causal-lm/smollm-135m-gguf-f16.json +++ b/testdata/golden/causal-lm/smollm-135m-gguf-f16.json @@ -57,7 +57,18 @@ "execution_provider": "cpu", "config_sha256": "134f95e6a635d978737d712ed61ac8959acebdf080eafae838cf97f12c416430", "preserve_quantization": false, - "keep_quantized": true + "keep_quantized": true, + "tokenizer": { + "repository": "HuggingFaceTB/SmolLM-135M", + "revision": "1d461723eec654e65efdc40cf49301c89c0c92f4", + "metadata_sha256": "46646ba36ecae43de6f9f649d217774b889e0fd405af92205319b882927493fc", + "identity_status": "exact", + "assets": [ + {"filename": "special_tokens_map.json", "size": 831, "sha256": "e786b595b9a23148bf1630df78d9037a048ea671e48bfd3549a1e3c233742bb3"}, + {"filename": "tokenizer.json", "size": 2104556, "sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c"}, + {"filename": "tokenizer_config.json", "size": 3685, "sha256": "238ad6b60d48e471624ea70bc79e92f2611844d5016471fee8c167854bcb98e8"} + ] + } } } } diff --git a/testdata/golden/causal-lm/smollm-135m-gguf-f16_generation.json b/testdata/golden/causal-lm/smollm-135m-gguf-f16_generation.json index dc2616077..54409346a 100644 --- a/testdata/golden/causal-lm/smollm-135m-gguf-f16_generation.json +++ b/testdata/golden/causal-lm/smollm-135m-gguf-f16_generation.json @@ -43,7 +43,18 @@ "execution_provider": "cpu", "config_sha256": "134f95e6a635d978737d712ed61ac8959acebdf080eafae838cf97f12c416430", "preserve_quantization": false, - "keep_quantized": true + "keep_quantized": true, + "tokenizer": { + "repository": "HuggingFaceTB/SmolLM-135M", + "revision": "1d461723eec654e65efdc40cf49301c89c0c92f4", + "metadata_sha256": "46646ba36ecae43de6f9f649d217774b889e0fd405af92205319b882927493fc", + "identity_status": "exact", + "assets": [ + {"filename": "special_tokens_map.json", "size": 831, "sha256": "e786b595b9a23148bf1630df78d9037a048ea671e48bfd3549a1e3c233742bb3"}, + {"filename": "tokenizer.json", "size": 2104556, "sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c"}, + {"filename": "tokenizer_config.json", "size": 3685, "sha256": "238ad6b60d48e471624ea70bc79e92f2611844d5016471fee8c167854bcb98e8"} + ] + } } } } diff --git a/testdata/golden/causal-lm/smollm2-135m-gguf-f16.json b/testdata/golden/causal-lm/smollm2-135m-gguf-f16.json index 905313546..16041b75e 100644 --- a/testdata/golden/causal-lm/smollm2-135m-gguf-f16.json +++ b/testdata/golden/causal-lm/smollm2-135m-gguf-f16.json @@ -57,7 +57,19 @@ "execution_provider": "cpu", "config_sha256": "c62123baf4e95656cdc9f5b798c14319bbaafec594526c462b10555f561969f9", "preserve_quantization": false, - "keep_quantized": true + "keep_quantized": true, + "tokenizer": { + "repository": "HuggingFaceTB/SmolLM2-135M-Instruct", + "revision": "12fd25f77366fa6b3b4b768ec3050bf629380bac", + "metadata_sha256": "cb0b637d59effdc3ab02f063039e597157fa4996663848cc2178510af5880ace", + "identity_status": "rejected", + "rejection_reason": "GGUF padding_token_id=0 conflicts with the pinned tokenizer pad_token_id=2.", + "assets": [ + {"filename": "special_tokens_map.json", "size": 655, "sha256": "2b7379f3ae813529281a5c602bc5a11c1d4e0a99107aaa597fe936c1e813ca52"}, + {"filename": "tokenizer.json", "size": 2104556, "sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c"}, + {"filename": "tokenizer_config.json", "size": 3764, "sha256": "4ec77d44f62efeb38d7e044a1db318f6a939438425312dfa333b8382dbad98df"} + ] + } } } } diff --git a/testdata/golden/causal-lm/smollm2-135m-gguf-f16_generation.json b/testdata/golden/causal-lm/smollm2-135m-gguf-f16_generation.json index 26ccb3d21..f0a7e883b 100644 --- a/testdata/golden/causal-lm/smollm2-135m-gguf-f16_generation.json +++ b/testdata/golden/causal-lm/smollm2-135m-gguf-f16_generation.json @@ -43,7 +43,19 @@ "execution_provider": "cpu", "config_sha256": "c62123baf4e95656cdc9f5b798c14319bbaafec594526c462b10555f561969f9", "preserve_quantization": false, - "keep_quantized": true + "keep_quantized": true, + "tokenizer": { + "repository": "HuggingFaceTB/SmolLM2-135M-Instruct", + "revision": "12fd25f77366fa6b3b4b768ec3050bf629380bac", + "metadata_sha256": "cb0b637d59effdc3ab02f063039e597157fa4996663848cc2178510af5880ace", + "identity_status": "rejected", + "rejection_reason": "GGUF padding_token_id=0 conflicts with the pinned tokenizer pad_token_id=2.", + "assets": [ + {"filename": "special_tokens_map.json", "size": 655, "sha256": "2b7379f3ae813529281a5c602bc5a11c1d4e0a99107aaa597fe936c1e813ca52"}, + {"filename": "tokenizer.json", "size": 2104556, "sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c"}, + {"filename": "tokenizer_config.json", "size": 3764, "sha256": "4ec77d44f62efeb38d7e044a1db318f6a939438425312dfa333b8382dbad98df"} + ] + } } } } diff --git a/tests/cli_test.py b/tests/cli_test.py index 1fd0bb46f..3612fa7fc 100644 --- a/tests/cli_test.py +++ b/tests/cli_test.py @@ -1040,14 +1040,14 @@ def test_invalid_runtime_value_errors(self): class TestCLIBuildGGUF: """The CLI must preserve the public GGUF architecture gate for every mode.""" - def test_runtime_deferred_graph_architecture_fails_before_output_creation( + def test_runtime_requires_explicit_pinned_tokenizer_before_output_creation( self, tmp_path: Path ) -> None: gguf_path = tmp_path / "llama.gguf" output_dir = tmp_path / "must-not-exist" _write_gated_gguf(gguf_path, architecture="llama", quantized=False) - with pytest.raises(SystemExit, match=r"runtime packaging for 'llama' is deferred"): + with pytest.raises(SystemExit, match="requires --tokenizer-repository"): main( [ "build-gguf", @@ -1062,6 +1062,34 @@ def test_runtime_deferred_graph_architecture_fails_before_output_creation( assert not output_dir.exists() + def test_runtime_rejects_mutable_tokenizer_revision_before_build( + self, tmp_path: Path + ) -> None: + gguf_path = tmp_path / "llama.gguf" + output_dir = tmp_path / "must-not-exist" + _write_gated_gguf(gguf_path, architecture="llama", quantized=False) + + with pytest.raises(SystemExit, match="immutable 40-hex"): + main( + [ + "build-gguf", + str(gguf_path), + "--output", + str(output_dir), + "--runtime", + "onnx-genai", + "--runtime-version", + "1.29.0", + "--tokenizer-repository", + "owner/tokenizer", + "--tokenizer-revision", + "main", + "--dequantize", + ] + ) + + assert not output_dir.exists() + @pytest.mark.parametrize( ("quantized", "options"), [ diff --git a/tests/e2e_golden_test.py b/tests/e2e_golden_test.py index a9d1a0f10..48afee047 100644 --- a/tests/e2e_golden_test.py +++ b/tests/e2e_golden_test.py @@ -3023,6 +3023,7 @@ def test_generation_matches_golden(self, case: GoldenTestCase) -> None: expected_token_ids = load_generation_golden(case) if expected_token_ids is None: pytest.skip(f"Generation golden file missing: {gen_path}") + _assert_gguf_golden_provenance(case, gen_path) tolerances = load_tolerances("L5", case.dtype) # Per-case tolerance override (e.g. VL multi-model pipeline has known diff --git a/tests/gguf_small_model_runtime_integration_test.py b/tests/gguf_small_model_runtime_integration_test.py index deed48731..43d81e2c3 100644 --- a/tests/gguf_small_model_runtime_integration_test.py +++ b/tests/gguf_small_model_runtime_integration_test.py @@ -29,7 +29,14 @@ from mobius import ModelPackage from mobius.__main__ import main +from mobius.integrations.gguf import ( + GGUFTokenizerAsset, + GGUFTokenizerSource, + materialize_gguf_tokenizer, + write_gguf_runtime_package, +) from mobius.integrations.gguf._reader import GGUFModel +from mobius.integrations.gguf._tokenizer import inspect_gguf_tokenizer @dataclass(frozen=True) @@ -46,6 +53,11 @@ class _RuntimeCase: tensor_qtypes: dict[str, int] config_sha256: str generated_tokens: tuple[int, ...] + tokenizer_repository: str + tokenizer_revision: str + tokenizer_metadata_sha256: str + tokenizer_assets: tuple[tuple[str, int, str], ...] + tokenizer_identity_exact: bool _CASES = ( @@ -83,6 +95,29 @@ class _RuntimeCase: 2428, 30, ), + tokenizer_repository="HuggingFaceTB/SmolLM-135M", + tokenizer_revision="1d461723eec654e65efdc40cf49301c89c0c92f4", + tokenizer_metadata_sha256=( + "46646ba36ecae43de6f9f649d217774b889e0fd405af92205319b882927493fc" + ), + tokenizer_assets=( + ( + "special_tokens_map.json", + 831, + "e786b595b9a23148bf1630df78d9037a048ea671e48bfd3549a1e3c233742bb3", + ), + ( + "tokenizer.json", + 2_104_556, + "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", + ), + ( + "tokenizer_config.json", + 3_685, + "238ad6b60d48e471624ea70bc79e92f2611844d5016471fee8c167854bcb98e8", + ), + ), + tokenizer_identity_exact=True, ), _RuntimeCase( name="smollm2-135m-instruct-f16", @@ -118,6 +153,29 @@ class _RuntimeCase: 253, 9154, ), + tokenizer_repository="HuggingFaceTB/SmolLM2-135M-Instruct", + tokenizer_revision="12fd25f77366fa6b3b4b768ec3050bf629380bac", + tokenizer_metadata_sha256=( + "cb0b637d59effdc3ab02f063039e597157fa4996663848cc2178510af5880ace" + ), + tokenizer_assets=( + ( + "special_tokens_map.json", + 655, + "2b7379f3ae813529281a5c602bc5a11c1d4e0a99107aaa597fe936c1e813ca52", + ), + ( + "tokenizer.json", + 2_104_556, + "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", + ), + ( + "tokenizer_config.json", + 3_764, + "4ec77d44f62efeb38d7e044a1db318f6a939438425312dfa333b8382dbad98df", + ), + ), + tokenizer_identity_exact=False, ), ) @@ -186,6 +244,16 @@ def test_small_f16_gguf_cli_full_logit_and_generation_parity( gguf_model = GGUFModel(gguf_path) qtypes = Counter(qtype.name for _, _, qtype, _ in gguf_model.tensor_items_raw()) assert dict(sorted(qtypes.items())) == case.tensor_qtypes + for filename, size, sha256 in case.tokenizer_assets: + asset_path = Path( + hf_hub_download( + repo_id=case.tokenizer_repository, + revision=case.tokenizer_revision, + filename=filename, + ) + ) + assert asset_path.stat().st_size == size + assert _sha256(asset_path) == sha256 output_dir = tmp_path / case.name captured: list[ModelPackage] = [] @@ -196,18 +264,31 @@ def capture_save(package: ModelPackage, *args: object, **kwargs: object) -> None original_save(package, *args, **kwargs) with mock.patch.object(ModelPackage, "save", capture_save): - main( - [ - "build-gguf", - str(gguf_path), - "--output", - str(output_dir), - "--dtype", - "f32", - "--execution-provider", - "cpu", - ] - ) + options = [ + "build-gguf", + str(gguf_path), + "--output", + str(output_dir), + "--dtype", + "f32", + "--execution-provider", + "cpu", + ] + if case.tokenizer_identity_exact: + options.extend( + [ + "--runtime", + "onnx-genai", + "--runtime-version", + "1.29.0", + "--tokenizer-repository", + case.tokenizer_repository, + "--tokenizer-revision", + case.tokenizer_revision, + "--local-files-only", + ] + ) + main(options) assert len(captured) == 1 route = json.loads(captured[0].gguf_import_route) @@ -236,10 +317,7 @@ def capture_save(package: ModelPackage, *args: object, **kwargs: object) -> None } package = ModelPackage.load(output_dir) assert tuple(package) == ("model",) - assert sorted(path.name for path in output_dir.iterdir()) == [ - "model.onnx", - "model.onnx.data", - ] + assert {"model.onnx", "model.onnx.data"} <= {path.name for path in output_dir.iterdir()} session = ort.InferenceSession( str(output_dir / "model.onnx"), providers=["CPUExecutionProvider"] ) @@ -252,6 +330,43 @@ def capture_save(package: ModelPackage, *args: object, **kwargs: object) -> None tokenizer = AutoTokenizer.from_pretrained( case.reference_repository, revision=case.reference_revision ) + if case.tokenizer_identity_exact: + packaged_tokenizer = AutoTokenizer.from_pretrained(output_dir, local_files_only=True) + assert packaged_tokenizer(case.prompt).input_ids == tokenizer(case.prompt).input_ids + else: + rejected_package = tmp_path / f"{case.name}-runtime" + with pytest.raises(ValueError, match="No unique GGUF runtime evidence"): + write_gguf_runtime_package( + captured[0], + gguf_path, + rejected_package, + runtime="onnx-genai", + runtime_version="1.29.0", + tokenizer_repository=case.tokenizer_repository, + tokenizer_revision=case.tokenizer_revision, + local_files_only=True, + ) + assert not rejected_package.exists() + source = GGUFTokenizerSource( + repository=case.tokenizer_repository, + revision=case.tokenizer_revision, + metadata_sha256=case.tokenizer_metadata_sha256, + assets=tuple(GGUFTokenizerAsset(*asset) for asset in case.tokenizer_assets), + ) + rejected_output = tmp_path / f"{case.name}-tokenizer" + assert ( + inspect_gguf_tokenizer(gguf_model.metadata, require_complete=True).metadata_sha256 + == case.tokenizer_metadata_sha256 + ) + with pytest.raises(ValueError, match="pad_token id differs from GGUF"): + materialize_gguf_tokenizer( + gguf_path, + rejected_output, + source=source, + metadata=gguf_model.metadata, + local_files_only=True, + ) + assert not rejected_output.exists() reference = AutoModelForCausalLM.from_pretrained( case.reference_repository, revision=case.reference_revision, diff --git a/tests/gguf_test.py b/tests/gguf_test.py index c6bb53ca7..80a54d713 100644 --- a/tests/gguf_test.py +++ b/tests/gguf_test.py @@ -607,10 +607,6 @@ def test_ort_genai_runtime_is_forwarded_to_package_writer(self, tmp_path): reason=None, ), ), - mock.patch( - "mobius.integrations.gguf._tokenizer.inspect_gguf_tokenizer", - return_value=mock.Mock(materialized=True), - ), mock.patch( "mobius.integrations.gguf.build_from_gguf", return_value=package, @@ -628,6 +624,10 @@ def test_ort_genai_runtime_is_forwarded_to_package_writer(self, tmp_path): str(output_dir), "--runtime", "ort-genai", + "--tokenizer-repository", + "owner/tokenizer", + "--tokenizer-revision", + "a" * 40, ] ) @@ -637,12 +637,15 @@ def test_ort_genai_runtime_is_forwarded_to_package_writer(self, tmp_path): str(output_dir), runtime="ort-genai", runtime_version=None, + tokenizer_repository="owner/tokenizer", + tokenizer_revision="a" * 40, + local_files_only=False, external_data="onnx", max_shard_size_bytes=None, max_workers=8, ) - def test_deferred_runtime_tokenizer_fails_before_graph_or_output(self, tmp_path): + def test_runtime_without_pinned_tokenizer_fails_before_graph_or_output(self, tmp_path): from mobius.__main__ import main from mobius.integrations.gguf._spec import Support @@ -665,14 +668,8 @@ def test_deferred_runtime_tokenizer_fails_before_graph_or_output(self, tmp_path) reason=None, ), ), - mock.patch( - "mobius.integrations.gguf._tokenizer.inspect_gguf_tokenizer", - return_value=mock.Mock( - materialized=False, reason="opaque pre-tokenizer is deferred" - ), - ), mock.patch("mobius.integrations.gguf.build_from_gguf") as build, - pytest.raises(SystemExit, match="opaque pre-tokenizer is deferred"), + pytest.raises(SystemExit, match="requires --tokenizer-repository"), ): main( [