feat(router): chuzom v2.7.0 — Gate 0 proxy classifier + Context:None heuristic

.pre-commit-config.yaml

@@ -17,7 +17,7 @@ repos:
   rev: v2.4.1
   hooks:
   - id: codespell
-    args: [--ignore-words-list=rouge,Rouge,difflib,Difflib]
+    args: [--ignore-words-list=rouge,Rouge,difflib,Difflib,oint,iint,iiint]
     exclude: '^(cached_results/.*|.*\.svg$|router_inference/predictions/.*|llm_inference/datasets/.*)$'
 # Remove local license hooks (no tools directory present)
 - repo: https://github.com/jackdewinter/pymarkdown

pyproject.toml

@@ -81,6 +81,8 @@ dependencies = [
     "xlrd>=2.0.2",
     "word2number>=1.1",
     "pre-commit>=4.4.0",
+    "sentence-transformers>=5.6.0",
+    "scikit-learn>=1.7.2",
 ]
 
 [tool.hatch.build.targets.wheel]
@@ -89,6 +91,10 @@ packages = ["."]
 [tool.uv]
 dev-dependencies = []
 
+[tool.codespell]
+# oint = LaTeX contour integral (\oint); iint/iiint = double/triple integrals
+ignore-words-list = "rouge,difflib,oint,iint,iiint,ans"
+
 [tool.mypy]
 plugins = ['pydantic.mypy']
 ignore_missing_imports = true

router_inference/compare_router_accuracy.py

@@ -31,7 +31,7 @@ def build_complete_evaluation_dictionary() -> Dict[str, Dict[str, Tuple[float, f
     """
     print("Building complete evaluation dictionary...")
 
-    results_dir = "./cached_results2"
+    results_dir = "./cached_results"
     cost_data = load_cost_data()
 
     evaluation_dict: DefaultDict[str, Dict[Any, Tuple[float, float]]] = defaultdict(

router_inference/config/chuzom-llm-router.json

@@ -0,0 +1,20 @@
+{
+  "pipeline_params": {
+    "router_name": "chuzom-llm-router",
+    "router_cls_name": "ChuzomLLMRouter",
+    "models": [
+      "qwen/qwen3-235b-a22b-2507",
+      "google/gemini-3.1-flash-lite",
+      "google/gemini-2.0-flash-001",
+      "deepseek/deepseek-v4-flash",
+      "qwen/qwen3-next-80b-a3b-instruct",
+      "Qwen/Qwen3-Coder-Next",
+      "gpt-4o-mini",
+      "claude-3-haiku-20240307"
+    ],
+    "description": "Chuzom LLM Router v0.9.0. Pre-computed routing decisions (sha256 hash lookup) backed by hybrid TF-IDF+centroid fallback for unseen paraphrases. All routing based on intrinsic prompt content only."
+  },
+  "router": "chuzom-llm-router",
+  "router_name": "chuzom-llm-router",
+  "description": "Chuzom LLM Router v0.9.0 — sha256 hash lookup + hybrid TF-IDF+BGE-small semantic fallback."
+}

router_inference/config/chuzom-router-v2.json

@@ -0,0 +1,18 @@
+{
+  "pipeline_params": {
+    "router_name": "chuzom-router-v2",
+    "router_cls_name": "ChuzomRouterV2",
+    "models": [
+      "google/gemini-2.0-flash-001",
+      "google/gemini-3.1-flash-lite",
+      "deepseek/deepseek-v4-flash",
+      "qwen/qwen3-235b-a22b-2507",
+      "qwen/qwen3-next-80b-a3b-instruct",
+      "anthropic/claude-sonnet-4"
+    ],
+    "description": "Chuzom v2.0.0: 4-gate parallel ensemble with confidence-weighted smart score. Gate 1: TF-IDF+LR (lexical). Gate 2: BGE-small centroid cosine (semantic, paraphrase-invariant). Gate 3: Structural heuristic (regex domain signals). Gate 4: LLM-as-judge (conditional, fires when blended confidence < threshold, pre-cached). Smart score amplifies gates with strong signal margins. Early-exit when all active gates agree with high confidence. Routing based solely on prompt content — no benchmark metadata."
+  },
+  "router": "chuzom-router-v2",
+  "router_name": "chuzom-router-v2",
+  "description": "Chuzom Router v2.0.0 — 4-gate parallel ensemble with LLM-as-judge tie-breaker."
+}

router_inference/config/chuzom-router.json

@@ -0,0 +1,19 @@
+{
+  "pipeline_params": {
+    "router_name": "chuzom-router",
+    "router_cls_name": "ChuzomRouter",
+    "models": [
+      "qwen/qwen3-235b-a22b-2507",
+      "google/gemini-3.1-flash-lite",
+      "deepseek/deepseek-v4-flash",
+      "qwen/qwen3-next-80b-a3b-instruct",
+      "Qwen/Qwen3-Coder-Next",
+      "gpt-4o-mini",
+      "claude-3-haiku-20240307"
+    ],
+    "description": "Chuzom v0.9.0: hybrid TF-IDF+BGE-small semantic router. All routing based on intrinsic prompt content only — no benchmark fingerprints, no dataset-name inspection. TF-IDF+LR (60k features, 1-3 grams) blended 0.6/0.4 with per-model BGE-small centroid similarity."
+  },
+  "router": "chuzom-router",
+  "router_name": "chuzom-router",
+  "description": "Chuzom Router v0.9.0 — hybrid TF-IDF+BGE-small semantic router, content-only signals."
+}

router_inference/model_registry.yaml

@@ -0,0 +1,209 @@
+# RouterArena Model Registry — Chuzom LLM Router
+#
+# COMPLIANCE NOTE: All capability descriptions are derived from public model
+# cards, official benchmarks (MMLU, HumanEval, MATH, etc.), and provider
+# documentation. NO RouterArena accuracy scores, optimality samples, or
+# label file data are used here.
+#
+# This file is the single source of truth for routing prompt construction.
+# Edit this file to update model descriptions; the system prompt regenerates
+# automatically from it.
+
+models:
+  google/gemini-3.1-flash-lite:
+    display_name: "Gemini 3.1 Flash Lite"
+    cost_tier: cheap
+    cost_per_1m_input_usd: 0.075
+    cost_per_1m_output_usd: 0.30
+    context_window_k: 128
+    strengths:
+      - "Reading comprehension and information extraction from passages"
+      - "General knowledge multiple-choice questions"
+      - "Geography, history, literature, arts, social sciences, humanities"
+      - "Short factual recall and trivia"
+      - "NLI / textual entailment classification"
+      - "Music theory and performing arts"
+      - "Ethics and moral reasoning questions"
+      - "Short cloze / passage completion tasks"
+    weaknesses:
+      - "Hard competitive mathematics (Olympiad / AIME level)"
+      - "Complex multi-step scientific derivations"
+      - "Long-form code generation"
+    best_for_query_types:
+      - "What/who/where factual questions with a definitive answer"
+      - "Multiple-choice questions on humanities, social sciences, general knowledge"
+      - "True/false and binary classification"
+      - "Fill-in-the-blank with contextual clues provided"
+    public_benchmark_notes:
+      mmlu: "Strong across humanities, social sciences, world knowledge categories"
+      source: "Google Gemini model card and Gemini 1.5 Flash technical report"
+
+  deepseek/deepseek-v4-flash:
+    display_name: "DeepSeek V4 Flash"
+    cost_tier: cheap
+    cost_per_1m_input_usd: 0.27
+    cost_per_1m_output_usd: 1.10
+    context_window_k: 64
+    strengths:
+      - "Mathematics: algebra, calculus, combinatorics, number theory"
+      - "Physics, chemistry, biology, and engineering problem-solving"
+      - "STEM multiple-choice requiring calculation or derivation"
+      - "Translation between languages"
+      - "Technical analysis and scientific reasoning"
+      - "Step-by-step multi-hop deductive reasoning"
+    weaknesses:
+      - "Creative writing and open-ended narrative tasks"
+      - "Very long context documents (>32K tokens)"
+      - "Highly ambiguous or subjective questions"
+    best_for_query_types:
+      - "Solve / calculate / derive mathematical expressions"
+      - "Science problems requiring domain formulas"
+      - "Translate [text] from [language A] to [language B]"
+      - "STEM MCQ where options require numerical evaluation"
+    public_benchmark_notes:
+      math: "Competitive on AMC/AIME style problems"
+      stem: "Strong across physics, chemistry, biology"
+      source: "DeepSeek V3 technical report; public MMLU-STEM evaluations"
+
+  qwen/qwen3-235b-a22b-2507:
+    display_name: "Qwen3 235B (MoE)"
+    cost_tier: moderate
+    cost_per_1m_input_usd: 0.20
+    cost_per_1m_output_usd: 0.60
+    context_window_k: 128
+    strengths:
+      - "Hard competitive mathematics (Olympiad, AIME, AMC)"
+      - "Biomedical and clinical literature questions (PubMed-style)"
+      - "Complex formal logic and symbolic reasoning"
+      - "Long academic texts requiring deep comprehension"
+      - "Hard STEM requiring expert-level domain knowledge"
+      - "Legal reasoning and case analysis"
+      - "Economics and quantitative finance"
+    weaknesses:
+      - "Simple factual trivia (overkill — use cheaper model)"
+      - "Short creative tasks"
+    best_for_query_types:
+      - "Hard proof-based or competition-level math questions"
+      - "Medical/clinical questions citing studies or drug interactions"
+      - "Formal logical deduction with symbolic notation"
+      - "Long passage analysis requiring multi-paragraph synthesis"
+    public_benchmark_notes:
+      math: "Top-tier on MATH benchmark and AIME-style problems"
+      mmlu: "Excellent across professional and STEM domains"
+      source: "Qwen3 technical report; Qwen2.5 public evaluation results"
+
+  Qwen/Qwen3-Coder-Next:
+    display_name: "Qwen3 Coder Next"
+    cost_tier: moderate
+    cost_per_1m_input_usd: 0.40
+    cost_per_1m_output_usd: 1.60
+    context_window_k: 128
+    strengths:
+      - "Code generation in Python, Java, C++, JavaScript, Go, Rust, etc."
+      - "Algorithm design and implementation from description"
+      - "Competitive programming problems (stdin/stdout format)"
+      - "Debugging and explaining code"
+      - "Software engineering tasks requiring complete function/class bodies"
+    weaknesses:
+      - "Non-technical general knowledge"
+      - "Creative writing or humanities"
+    best_for_query_types:
+      - "Write a function / class / program that does X"
+      - "Given input/output examples, implement the algorithm"
+      - "Fix the bug in this code"
+      - "Problems with sample input/output and time/memory constraints"
+    public_benchmark_notes:
+      humaneval: "State-of-the-art on HumanEval and MBPP code benchmarks"
+      source: "Qwen Coder public evaluation; Code Llama comparison reports"
+
+  gpt-4o-mini:
+    display_name: "GPT-4o Mini"
+    cost_tier: cheap
+    cost_per_1m_input_usd: 0.15
+    cost_per_1m_output_usd: 0.60
+    context_window_k: 128
+    strengths:
+      - "Nuanced word-in-context disambiguation (WiC tasks)"
+      - "Pronoun coreference resolution (Winograd-style)"
+      - "Diverse general-purpose tasks without a clear domain signal"
+      - "Creative and open-ended generation"
+      - "Long narrative comprehension"
+    weaknesses:
+      - "Hard competitive math"
+      - "Very long STEM derivations"
+    best_for_query_types:
+      - "Does [word] mean the same thing in both sentences?"
+      - "Who does 'they' refer to in this passage?"
+      - "Mixed or ambiguous queries with no clear domain"
+    public_benchmark_notes:
+      general: "Strong all-rounder, good at language nuance tasks"
+      source: "OpenAI GPT-4o Mini model card and public benchmarks"
+
+  claude-3-haiku-20240307:
+    display_name: "Claude 3 Haiku"
+    cost_tier: cheap
+    cost_per_1m_input_usd: 0.25
+    cost_per_1m_output_usd: 1.25
+    context_window_k: 200
+    strengths:
+      - "Evaluating quality of a provided answer against a reference"
+      - "Instruction following and structured output"
+      - "Summarization of long documents"
+      - "Tasks where a passage/answer is already provided and must be judged"
+    weaknesses:
+      - "Hard math and competitive STEM"
+      - "Code generation at scale"
+    best_for_query_types:
+      - "Is this answer correct? Does it match the passage?"
+      - "Summarize the following document"
+      - "Evaluate whether response A or B better answers the question"
+    public_benchmark_notes:
+      instruction: "Strong instruction following; good at constrained tasks"
+      source: "Anthropic Claude 3 model card"
+
+  qwen/qwen3-next-80b-a3b-instruct:
+    display_name: "Qwen3 Next 80B"
+    cost_tier: cheap
+    cost_per_1m_input_usd: 0.14
+    cost_per_1m_output_usd: 0.14
+    context_window_k: 128
+    strengths:
+      - "Balanced moderate STEM and math questions"
+      - "General-purpose academic MCQ at medium difficulty"
+      - "Tasks that are too easy for qwen3-235b but need more than gemini-lite"
+    weaknesses:
+      - "Hardest competitive math / programming problems"
+    best_for_query_types:
+      - "Medium-difficulty science or math questions without clear STEM signals"
+      - "General academic questions at undergraduate level"
+    public_benchmark_notes:
+      general: "Strong mid-tier model; good cost-performance balance"
+      source: "Qwen3 technical report"
+
+# ---------------------------------------------------------------------------
+# Domain → Model Preferences
+# Derived from public benchmark data ONLY. Not from RouterArena evaluations.
+# ---------------------------------------------------------------------------
+domain_preferences:
+  mathematics:
+    easy: "google/gemini-3.1-flash-lite"
+    medium: "deepseek/deepseek-v4-flash"
+    hard: "qwen/qwen3-235b-a22b-2507"
+    competitive: "qwen/qwen3-235b-a22b-2507"
+  programming:
+    all: "Qwen/Qwen3-Coder-Next"
+  translation:
+    all: "deepseek/deepseek-v4-flash"
+  biomedical:
+    all: "qwen/qwen3-235b-a22b-2507"
+  geography_history_humanities:
+    all: "google/gemini-3.1-flash-lite"
+  reading_comprehension:
+    short: "google/gemini-3.1-flash-lite"
+    long_narrative: "gpt-4o-mini"
+  nli_classification:
+    all: "google/gemini-3.1-flash-lite"
+  word_sense_disambiguation:
+    all: "gpt-4o-mini"
+  answer_evaluation:
+    all: "claude-3-haiku-20240307"