diff --git a/.env.example b/.env.example index ac259be..8139003 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,19 @@ +# ============================================================================= +# SERVER-SIDE environment template (FastAPI backend + local dev tooling). +# ============================================================================= +# Loaded ONLY by the backend (see backend/app/config.py) and local dev scripts. +# Copy to `.env` and fill in real values. `.env` is gitignored and MUST NEVER +# be committed or bundled into the Flutter app. +# +# The Flutter client does NOT read this file. Its only bundled configuration is +# assets/config/app_config.env, which contains PUBLIC values only (API base +# URL, non-secret speech settings) — never DB credentials or API keys. The +# device reaches the database exclusively through the FastAPI backend over +# HTTPS; provider API keys are used server-side (e.g. the voice-transcribe +# proxy), never shipped to the device. +# ============================================================================= + +# --- Database (server-side ONLY; never bundled in the app) ------------------- DB_HOST= DB_PORT= DB_NAME= @@ -6,6 +22,20 @@ DB_PASSWORD= DB_USE_SSL= ENTITY_SEARCH_FALLBACK_MODE= +# --- Observability ----------------------------------------------------------- +# Structured (JSON) request timing lines are always emitted; these tune them. +LOG_FORMAT=json +LOG_LEVEL=INFO +# Attaches the per-phase timing breakdown to the search response body. Leave +# off in production and correlate via the request id in the logs instead. +EXPOSE_DEBUG_TIMINGS=false + +# --- Database connection pool ------------------------------------------------ +DB_POOL_SIZE=5 +DB_MAX_OVERFLOW=5 +DB_POOL_TIMEOUT_SECONDS=10 +DB_POOL_RECYCLE_SECONDS=1800 + QUERY_UNDERSTANDING_PROVIDER=gemini QUERY_UNDERSTANDING_MODEL=gemini-3.5-flash-lite GEMINI_API_KEY=your_api_key_here diff --git a/assets/config/app_config.env b/assets/config/app_config.env new file mode 100644 index 0000000..9ac2b9b --- /dev/null +++ b/assets/config/app_config.env @@ -0,0 +1,26 @@ +# ============================================================================= +# PUBLIC CLIENT CONFIGURATION — ships inside the app bundle. +# ============================================================================= +# This file is compiled into the Flutter app and is readable by anyone who +# installs it. It MUST contain ONLY non-sensitive, public configuration. +# +# NEVER put secrets here (or in any client-bundled file): +# - No DB_HOST / DB_PORT / DB_USER / DB_PASSWORD / DATABASE_URL +# - No OPENAI_API_KEY / GEMINI_API_KEY / ANTHROPIC_API_KEY +# - No AWS keys, tokens, or passwords +# +# The device never connects to the database directly. All data access goes +# through the FastAPI backend over HTTPS. Database credentials and provider +# API keys live ONLY in server-side environment variables (see backend/.env +# / .env.example, loaded by backend/app/config.py). +# ============================================================================= + +# Base URL of the FastAPI backend. This is the single piece of config the +# client needs. Override per build/flavor as appropriate (public value). +PYTHON_BACKEND_BASE_URL=http://0.0.0.0:8765/ + +# Speech provider selection (non-secret). Cloud transcription is proxied +# through the backend's /v1/voice/transcribe endpoint, so no API key is +# required on the device. +SPEECH_PROVIDER=openai_proxy +SPEECH_MODEL=whisper-1 diff --git a/backend/app/api/v1/conversation.py b/backend/app/api/v1/conversation.py index 5718218..6ef3d5d 100644 --- a/backend/app/api/v1/conversation.py +++ b/backend/app/api/v1/conversation.py @@ -1,8 +1,11 @@ +from functools import lru_cache + from fastapi import APIRouter, Depends from pydantic import BaseModel, ConfigDict, Field from sqlalchemy.ext.asyncio import AsyncConnection from ...config import Settings, get_settings +from ...observability import Phase, current_timings from ...db.engine import get_connection from ...domain.conversation_session import SearchConversationSession from ...entity_search.adapter import EntitySearchLookupAdapter @@ -13,7 +16,11 @@ from ...query_understanding.providers import AnthropicProvider, GeminiProvider, MockProvider, OpenAIProvider from ...query_understanding.service import QueryUnderstandingService from ...repositories.search_repository import SearchRepository -from ...services.conversational_search_service import ConversationalSearchResult, ConversationalSearchService +from ...services.conversational_search_service import ( + ConversationalSearchResult, + ConversationalSearchService, + record_result_attributes, +) router = APIRouter(prefix="/v1/conversation") @@ -33,6 +40,30 @@ class ConversationSearchResponse(BaseModel): session: SearchConversationSession result: ConversationalSearchResult request_id: int | None = Field(default=None, alias="requestId") + trace_id: str | None = Field(default=None, alias="traceId") + + +@lru_cache(maxsize=4) +def _cached_query_service( + provider: str, + model: str, + api_key: str | None, + base_url: str | None, + temperature: float, + timeout_seconds: float, + max_retries: int, + provider_type: type, +) -> QueryUnderstandingService: + config = ProviderConfig( + provider=provider, + model=model, + api_key=api_key, + base_url=base_url, + temperature=temperature, + timeout_seconds=timeout_seconds, + max_retries=max_retries, + ) + return QueryUnderstandingService(provider_type(config)) def _build_query_service(settings: Settings) -> QueryUnderstandingService: @@ -83,16 +114,19 @@ def _build_query_service(settings: Settings) -> QueryUnderstandingService: "missing. Provide the API key or change the provider." ) - config = ProviderConfig( - provider=name, - model=settings.query_understanding_model, - api_key=api_key, - base_url=urls.get(name), - temperature=settings.query_understanding_temperature, - timeout_seconds=settings.query_understanding_timeout_seconds, - max_retries=settings.query_understanding_max_retries, + # Validation above runs on every call and still fails fast; only the + # construction of the (stateless) service and provider is memoised, so + # each request no longer rebuilds them. + return _cached_query_service( + name, + settings.query_understanding_model, + api_key, + urls.get(name), + settings.query_understanding_temperature, + settings.query_understanding_timeout_seconds, + settings.query_understanding_max_retries, + provider_types[name], ) - return QueryUnderstandingService(provider_types[name](config)) from ...entity_search.warmup import ( @@ -122,14 +156,28 @@ async def get_conversational_service( async def search_conversation( request: ConversationSearchRequest, service: ConversationalSearchService = Depends(get_conversational_service), + settings: Settings = Depends(get_settings), ) -> ConversationSearchResponse: + timings = current_timings() + if timings is not None: + # Everything before the handler body — dependency resolution, which is + # where the DB connection is checked out and the entity index awaited. + timings.add( + Phase.DEPENDENCIES, + max(0.0, timings.total_ms - sum(timings.phases.values())), + ) updated_session, result = await service.search( request.query, session=request.session, language=request.language, + expose_timings=settings.expose_debug_timings, ) + if timings is not None: + record_result_attributes(timings, result) + timings.mark_handler_complete() return ConversationSearchResponse( session=updated_session, result=result, request_id=request.request_id, + trace_id=result.request_id, ) diff --git a/backend/app/config.py b/backend/app/config.py index ba8abff..c85a765 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -43,6 +43,16 @@ def _apply_env_fallbacks(cls, data: Any) -> Any: db_user: str = "" db_password: SecretStr = SecretStr("") db_use_ssl: bool = False + # Connection pooling. Sized for a small Railway container: enough warm + # connections to absorb bursts without exhausting MySQL's max_connections + # across workers. `recycle` stays well under a typical 8h wait_timeout. + db_pool_size: int = 5 + db_max_overflow: int = 5 + db_pool_timeout_seconds: float = 10.0 + db_pool_recycle_seconds: int = 1800 + # Per-request phase timings are attached to the response only when this is + # on. Production correlates via request_id in structured logs instead. + expose_debug_timings: bool = False entity_search_fallback_mode: str = "FALLBACK" query_understanding_provider: str = "mock" query_understanding_model: str = "mock-parser-v1" diff --git a/backend/app/db/engine.py b/backend/app/db/engine.py index c73feec..edfed91 100644 --- a/backend/app/db/engine.py +++ b/backend/app/db/engine.py @@ -1,20 +1,62 @@ from collections.abc import AsyncIterator + from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, create_async_engine -from sqlalchemy.pool import NullPool + from ..config import get_settings +from ..observability import Phase, current_timings _engine: AsyncEngine | None = None + def get_engine() -> AsyncEngine: global _engine if _engine is None: settings = get_settings() if not settings.db_host or not settings.db_name or not settings.db_user: raise RuntimeError("Database configuration is incomplete") - # Async connections must never leak across worker/event-loop boundaries. - _engine = create_async_engine(settings.database_url, poolclass=NullPool) + # A pooled engine. The previous NullPool opened a fresh TCP + TLS + + # MySQL auth handshake on every request, measured at ~178 ms each + # against the production host; a checked-out pooled connection costs + # ~30 ms. `pool_pre_ping` covers connections the server closed while + # idle, and `pool_recycle` stays below the usual wait_timeout so a + # stale connection is never handed to a request. + # + # Async connections must still never leak across event loops: the + # engine is created lazily per process, and every worker process + # therefore builds its own pool. + _engine = create_async_engine( + settings.database_url, + pool_size=settings.db_pool_size, + max_overflow=settings.db_max_overflow, + pool_timeout=settings.db_pool_timeout_seconds, + pool_recycle=settings.db_pool_recycle_seconds, + pool_pre_ping=True, + ) return _engine + +async def dispose_engine() -> None: + global _engine + if _engine is not None: + await _engine.dispose() + _engine = None + + async def get_connection() -> AsyncIterator[AsyncConnection]: - async with get_engine().connect() as connection: + """Yields a pooled connection, recording checkout cost as `db_connect`. + + Dependency resolution runs before the handler body, so without this + measurement the connection cost lands outside every in-handler timer. + """ + timings = current_timings() + if timings is None: + async with get_engine().connect() as connection: + yield connection + return + with timings.measure(Phase.DB_CONNECT): + context = get_engine().connect() + connection = await context.__aenter__() + try: yield connection + finally: + await context.__aexit__(None, None, None) diff --git a/backend/app/main.py b/backend/app/main.py index d30fd80..9c9a0a0 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -10,16 +10,23 @@ from .api.v1.query_understanding import router as query_understanding_router from .api.v1.search import router as search_router from .api.v1.voice import router as voice_router -from .db.engine import get_engine +from .db.engine import dispose_engine, get_engine from .domain.errors import ApiError, ErrorCode from .entity_search.warmup import ( cancel_background_entity_search_warmup, start_background_entity_search_warmup, ) +from .observability.logging import configure_logging +from .observability.middleware import timing_middleware +from .query_understanding.providers.http import close_client @asynccontextmanager async def lifespan(app: FastAPI): + # Lift app loggers above uvicorn's default config. Without this the + # OpenEntity warm-up and per-request timing lines never reach stdout, which + # is why cold-start cost was previously invisible in production. + configure_logging() # Non-blocking OpenEntity background warm-up engine = get_engine() start_background_entity_search_warmup(engine) @@ -27,9 +34,12 @@ async def lifespan(app: FastAPI): yield finally: await cancel_background_entity_search_warmup() + await close_client() + await dispose_engine() app = FastAPI(title="AI Rally Search deterministic backend", version="0.1.0", lifespan=lifespan) +app.middleware("http")(timing_middleware) app.include_router(health_router) app.include_router(search_router) app.include_router(offline_router) diff --git a/backend/app/observability/__init__.py b/backend/app/observability/__init__.py new file mode 100644 index 0000000..b2da4b2 --- /dev/null +++ b/backend/app/observability/__init__.py @@ -0,0 +1,23 @@ +"""Latency observability: request identity, phase timings, structured logs. + +Nothing in here records raw user text, credentials, or result payloads. The +only free-form value that ever reaches a log line is the request id, which the +client generates and which carries no user data. +""" + +from .request_context import ( + current_request_id, + current_timings, + new_request_id, + request_scope, +) +from .timings import Phase, RequestTimings + +__all__ = [ + "Phase", + "RequestTimings", + "current_request_id", + "current_timings", + "new_request_id", + "request_scope", +] diff --git a/backend/app/observability/logging.py b/backend/app/observability/logging.py new file mode 100644 index 0000000..aaae6e9 --- /dev/null +++ b/backend/app/observability/logging.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import json +import logging +import os +import sys +from typing import Any + +TIMING_LOGGER = "app.latency" + +_logger = logging.getLogger(TIMING_LOGGER) + +# Keys that may never appear in a timing record, whatever a caller passes. +# Raw query text is the important one: timing lines are shipped to whatever +# aggregator the platform provides, and search text is user content. +_FORBIDDEN_KEYS = frozenset( + {"query", "raw_query", "text", "transcript", "api_key", "authorization", "password", "session"} +) + + +class JsonFormatter(logging.Formatter): + """One JSON object per line, so timing records stay machine-queryable.""" + + def format(self, record: logging.LogRecord) -> str: + payload: dict[str, Any] = { + "level": record.levelname, + "logger": record.name, + "message": record.getMessage(), + } + extra = getattr(record, "timing", None) + if isinstance(extra, dict): + payload.update(extra) + if record.exc_info: + payload["exc"] = self.formatException(record.exc_info) + return json.dumps(payload, default=str, separators=(",", ":")) + + +def structured_logs_enabled() -> bool: + return os.getenv("LOG_FORMAT", "json").strip().lower() == "json" + + +def configure_logging() -> None: + """Installs the JSON handler and lifts app loggers to INFO. + + Without this the warm-up and timing loggers are silently swallowed by + uvicorn's default config, which is why cold-start cost was previously + invisible in production logs. + """ + level = getattr(logging, os.getenv("LOG_LEVEL", "INFO").strip().upper(), logging.INFO) + handler = logging.StreamHandler(sys.stdout) + if structured_logs_enabled(): + handler.setFormatter(JsonFormatter()) + else: + handler.setFormatter(logging.Formatter("%(levelname)s %(name)s %(message)s")) + root = logging.getLogger("app") + root.handlers = [handler] + root.setLevel(level) + root.propagate = False + + +def sanitize(record: dict[str, Any]) -> dict[str, Any]: + return {k: v for k, v in record.items() if k.lower() not in _FORBIDDEN_KEYS} + + +def log_request_timing(record: dict[str, Any], *, message: str = "search_timing") -> None: + """Emits one structured timing line. + + Overhead is a dict copy and a `json.dumps` of ~15 small scalars, on the + order of tens of microseconds against a request measured in hundreds of + milliseconds. + """ + if not _logger.isEnabledFor(logging.INFO): + return + _logger.info(message, extra={"timing": sanitize(record)}) diff --git a/backend/app/observability/middleware.py b/backend/app/observability/middleware.py new file mode 100644 index 0000000..cf913e6 --- /dev/null +++ b/backend/app/observability/middleware.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import time +from collections.abc import Awaitable, Callable + +from fastapi import Request, Response + +from .logging import log_request_timing +from .request_context import request_scope + +REQUEST_ID_HEADER = "X-Request-Id" +SERVER_TIMING_HEADER = "X-Backend-Total-Ms" + + +async def timing_middleware( + request: Request, call_next: Callable[[Request], Awaitable[Response]] +) -> Response: + """Wraps every request in a timing scope. + + This is the only place that measures the *true* backend total: it starts + before FastAPI resolves dependencies (where the DB connection is opened) + and stops after the response is rendered, so dependency and serialization + cost can no longer hide outside the reported number. + """ + started = time.perf_counter() + header_id = request.headers.get(REQUEST_ID_HEADER) + with request_scope(header_id) as timings: + timings.update(path=request.url.path, method=request.method) + request.state.timings = timings + request.state.request_id = timings.request_id + try: + response = await call_next(request) + except Exception: + total_ms = (time.perf_counter() - started) * 1000 + timings.set("outcome", "exception") + log_request_timing(timings.snapshot(total_ms=total_ms)) + raise + timings.close_serialization() + total_ms = (time.perf_counter() - started) * 1000 + timings.set("status_code", response.status_code) + record = timings.snapshot(total_ms=total_ms) + log_request_timing(record) + response.headers[REQUEST_ID_HEADER] = timings.request_id + response.headers[SERVER_TIMING_HEADER] = f"{total_ms:.1f}" + return response diff --git a/backend/app/observability/request_context.py b/backend/app/observability/request_context.py new file mode 100644 index 0000000..46ffd6b --- /dev/null +++ b/backend/app/observability/request_context.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import re +import uuid +from contextlib import contextmanager +from contextvars import ContextVar +from typing import Iterator + +from .timings import RequestTimings + +_request_id: ContextVar[str | None] = ContextVar("rally_request_id", default=None) +_timings: ContextVar[RequestTimings | None] = ContextVar("rally_timings", default=None) + +# A client-supplied correlation id is echoed into logs, so it is constrained to +# an opaque, bounded token. Anything else is replaced with a generated id +# rather than rejected: correlation is a convenience, never a request gate. +_SAFE_REQUEST_ID = re.compile(r"^[A-Za-z0-9._:-]{1,64}$") + + +def new_request_id() -> str: + return uuid.uuid4().hex + + +def sanitize_request_id(candidate: str | None) -> str: + if candidate and _SAFE_REQUEST_ID.match(candidate): + return candidate + return new_request_id() + + +def current_request_id() -> str | None: + return _request_id.get() + + +def current_timings() -> RequestTimings | None: + """The active collector, or None outside a request scope. + + Callers deep in the pipeline use this instead of threading a timings + argument through every signature; when no scope is active (unit tests, + scripts) they simply record nothing. + """ + return _timings.get() + + +@contextmanager +def request_scope(request_id: str | None = None) -> Iterator[RequestTimings]: + rid = sanitize_request_id(request_id) + timings = RequestTimings(request_id=rid) + id_token = _request_id.set(rid) + timings_token = _timings.set(timings) + try: + yield timings + finally: + _timings.reset(timings_token) + _request_id.reset(id_token) diff --git a/backend/app/observability/timings.py b/backend/app/observability/timings.py new file mode 100644 index 0000000..ad76166 --- /dev/null +++ b/backend/app/observability/timings.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import time +from contextlib import contextmanager +from dataclasses import dataclass, field +from enum import StrEnum +from typing import Any, Iterator + + +class Phase(StrEnum): + """The phases of one search request, in pipeline order. + + Every value maps 1:1 onto a `_ms` key in the emitted timing record, + so adding a phase here is the only edit needed to surface a new number. + """ + + # Time spent in FastAPI dependency resolution before the handler body runs. + # This is where the per-request DB connection is opened, so it must be + # measured separately or it disappears from the breakdown entirely. + DEPENDENCIES = "dependencies" + DB_CONNECT = "db_connect" + QUERY_UNDERSTANDING = "query_understanding" + GEMINI = "gemini" + EXTERNAL_API = "external_api" + DETERMINISTIC_RECOVERY = "deterministic_recovery" + ENTITY_RESOLUTION = "entity_resolution" + SEARCH_PLAN = "search_plan" + REPOSITORY_DB = "repository_db" + SERIALIZATION = "serialization" + + +@dataclass +class RequestTimings: + """Accumulates per-phase durations for one request. + + Durations accumulate, so a phase entered twice (a provider retry, two DB + round-trips) reports the summed time rather than only the last span. The + collector is deliberately allocation-light: measuring adds two + `perf_counter` reads and a float add per phase. + """ + + request_id: str + started: float = field(default_factory=time.perf_counter) + phases: dict[str, float] = field(default_factory=dict) + attributes: dict[str, Any] = field(default_factory=dict) + _handler_done: float | None = field(default=None, repr=False) + + def add(self, phase: Phase | str, duration_ms: float) -> None: + key = str(phase) + self.phases[key] = self.phases.get(key, 0.0) + max(0.0, duration_ms) + + @contextmanager + def measure(self, phase: Phase | str) -> Iterator[None]: + started = time.perf_counter() + try: + yield + finally: + self.add(phase, (time.perf_counter() - started) * 1000) + + def mark_handler_complete(self) -> None: + """Marks the instant the handler returned its response model. + + Response serialization happens after that point but still inside the + request, so the middleware turns this mark into `serialization_ms`. + """ + self._handler_done = time.perf_counter() + + def close_serialization(self) -> None: + done = getattr(self, "_handler_done", None) + if done is not None: + self.add(Phase.SERIALIZATION, (time.perf_counter() - done) * 1000) + + def set(self, key: str, value: Any) -> None: + """Records a non-timing attribute (model used, intent, flags). + + Values are expected to be small scalars. Raw query text must never be + passed here — see the module docstring in `observability/__init__`. + """ + self.attributes[key] = value + + def update(self, **values: Any) -> None: + self.attributes.update(values) + + @property + def total_ms(self) -> float: + return (time.perf_counter() - self.started) * 1000 + + def snapshot(self, *, total_ms: float | None = None) -> dict[str, Any]: + """The flat record written to logs and (in debug mode) the response.""" + record: dict[str, Any] = {"request_id": self.request_id} + record["total_backend_ms"] = round( + self.total_ms if total_ms is None else total_ms, 1 + ) + for phase in Phase: + value = self.phases.get(str(phase)) + if value is not None: + record[f"{phase}_ms"] = round(value, 1) + record.update(self.attributes) + return record diff --git a/backend/app/query_understanding/providers/gemini_provider.py b/backend/app/query_understanding/providers/gemini_provider.py index 342c9e2..a4edbca 100644 --- a/backend/app/query_understanding/providers/gemini_provider.py +++ b/backend/app/query_understanding/providers/gemini_provider.py @@ -1,13 +1,20 @@ -import json +from functools import lru_cache from typing import Any -from urllib.parse import quote +from ...observability import Phase from ..models import ProviderResponse, TokenUsage from ..prompt import SYSTEM_PROMPT from ..provider import ProviderError, QueryUnderstandingProvider from .http import post_json +@lru_cache(maxsize=1) +def _response_schema() -> dict[str, Any]: + """The SearchQuery JSON schema, built once per process rather than per call.""" + from ...domain.search_query import SearchQuery + + return SearchQuery.model_json_schema(by_alias=True) + class GeminiProvider(QueryUnderstandingProvider): async def parse_raw( @@ -21,23 +28,47 @@ async def parse_raw( if not c.api_key: raise ProviderError("GEMINI_API_KEY is missing") model = c.model if c.model.startswith("models/") else f"models/{c.model}" - generation = {"temperature": c.temperature, "maxOutputTokens": c.max_tokens, "responseMimeType": "application/json"} + generation = { + "temperature": c.temperature, + "maxOutputTokens": c.max_tokens, + "responseMimeType": "application/json", + } if c.structured_output: - from ...domain.search_query import SearchQuery - generation["responseJsonSchema"] = SearchQuery.model_json_schema(by_alias=True) + generation["responseJsonSchema"] = _response_schema() generation.update(c.parameters) user_content = natural_language_query if context is not None: ctx_str = getattr(context, "format_prompt_context", lambda: "")() if ctx_str: user_content = f"{ctx_str}{natural_language_query}" - body = {"systemInstruction": {"parts": [{"text": SYSTEM_PROMPT}]}, "contents": [{"role": "user", "parts": [{"text": user_content}]}], "generationConfig": generation} + body = { + "systemInstruction": {"parts": [{"text": SYSTEM_PROMPT}]}, + "contents": [{"role": "user", "parts": [{"text": user_content}]}], + "generationConfig": generation, + } - url = f"{(c.base_url or 'https://generativelanguage.googleapis.com/v1beta').rstrip('/')}/{model}:generateContent?key={quote(c.api_key)}" - data = await post_json(url, body, {"Content-Type": "application/json"}, c.timeout_seconds) + base = (c.base_url or "https://generativelanguage.googleapis.com/v1beta").rstrip("/") + # The key travels as a header rather than a query parameter so it + # cannot be captured by URL logging on any hop. + data = await post_json( + f"{base}/{model}:generateContent", + body, + {"Content-Type": "application/json", "x-goog-api-key": c.api_key}, + c.timeout_seconds, + phase=Phase.GEMINI, + ) try: raw = data["candidates"][0]["content"]["parts"][0]["text"] except (KeyError, IndexError, TypeError) as exc: raise ProviderError("Gemini response contained no candidate text") from exc u = data.get("usageMetadata", {}) - return ProviderResponse(raw_response=raw, usage=TokenUsage(input_tokens=u.get("promptTokenCount"), output_tokens=u.get("candidatesTokenCount"), cached_tokens=u.get("cachedContentTokenCount"), total_tokens=u.get("totalTokenCount")), metadata={"finish_reason": data["candidates"][0].get("finishReason")}) + return ProviderResponse( + raw_response=raw, + usage=TokenUsage( + input_tokens=u.get("promptTokenCount"), + output_tokens=u.get("candidatesTokenCount"), + cached_tokens=u.get("cachedContentTokenCount"), + total_tokens=u.get("totalTokenCount"), + ), + metadata={"finish_reason": data["candidates"][0].get("finishReason")}, + ) diff --git a/backend/app/query_understanding/providers/http.py b/backend/app/query_understanding/providers/http.py index bdf9cb2..177f636 100644 --- a/backend/app/query_understanding/providers/http.py +++ b/backend/app/query_understanding/providers/http.py @@ -1,23 +1,87 @@ +from __future__ import annotations + import asyncio import json -import urllib.error -import urllib.request from typing import Any +import httpx + +from ...observability import Phase, current_timings from ..provider import ProviderError, ProviderTimeout +# One process-wide client, so provider calls reuse a warm TLS connection. +# +# This replaces a `urllib.request.urlopen` call dispatched to a worker thread, +# which established a fresh DNS + TCP + TLS connection for every single model +# call. Measured against the production Gemini endpoint, that per-call +# handshake had a pathological tail: over 25 sequential calls the p50 was +# ~500 ms but 2 calls exceeded 20 s, i.e. ~8% of model calls blew straight +# through any 4 s budget. The same 25 calls over a keep-alive httpx client had +# a max of 729 ms. Connection reuse is the fix; the timeout below is the +# backstop. +_client: httpx.AsyncClient | None = None +_client_lock = asyncio.Lock() + +_LIMITS = httpx.Limits( + max_connections=20, + max_keepalive_connections=10, + keepalive_expiry=60.0, +) + + +async def get_client() -> httpx.AsyncClient: + global _client + if _client is None or _client.is_closed: + async with _client_lock: + if _client is None or _client.is_closed: + _client = httpx.AsyncClient(limits=_LIMITS, http2=False) + return _client + + +async def close_client() -> None: + global _client + if _client is not None and not _client.is_closed: + await _client.aclose() + _client = None + + +async def post_json( + url: str, + body: dict[str, Any], + headers: dict[str, str], + timeout: float, + *, + params: dict[str, str] | None = None, + phase: Phase | str = Phase.EXTERNAL_API, +) -> dict[str, Any]: + """POSTs JSON over the shared pooled client. + + `timeout` is applied per-phase (connect/read/write) rather than to the + whole operation, so a stalled connect fails fast instead of consuming the + entire budget before the request is even sent. + """ + client = await get_client() + timings = current_timings() + limits = httpx.Timeout(timeout, connect=min(timeout, 10.0)) + try: + if timings is not None: + with timings.measure(phase): + response = await client.post( + url, params=params, json=body, headers=headers, timeout=limits + ) + else: + response = await client.post( + url, params=params, json=body, headers=headers, timeout=limits + ) + except httpx.TimeoutException as exc: + raise ProviderTimeout(str(exc) or "provider request timed out") from exc + except httpx.HTTPError as exc: + raise ProviderError(str(exc)) from exc -async def post_json(url: str, body: dict[str, Any], headers: dict[str, str], timeout: float) -> dict[str, Any]: - def send() -> dict[str, Any]: - request = urllib.request.Request(url, data=json.dumps(body).encode(), headers=headers, method="POST") - try: - with urllib.request.urlopen(request, timeout=timeout) as response: - return json.loads(response.read().decode()) - except urllib.error.HTTPError as exc: - detail = exc.read().decode(errors="replace") - raise ProviderError(f"HTTP {exc.code}: {detail[:1000]}") from exc - except TimeoutError as exc: - raise ProviderTimeout(str(exc)) from exc - except (urllib.error.URLError, json.JSONDecodeError) as exc: - raise ProviderError(str(exc)) from exc - return await asyncio.to_thread(send) + if response.status_code < 200 or response.status_code >= 300: + detail = response.text + raise ProviderError(f"HTTP {response.status_code}: {detail[:1000]}") + try: + return response.json() + except (json.JSONDecodeError, ValueError) as exc: + raise ProviderError(f"provider returned non-JSON body: {exc}") from exc diff --git a/backend/app/services/conversational_search_service.py b/backend/app/services/conversational_search_service.py index 6319fa8..68fa966 100644 --- a/backend/app/services/conversational_search_service.py +++ b/backend/app/services/conversational_search_service.py @@ -13,6 +13,7 @@ from ..domain.search_query import SearchQuery from ..domain.summary import generate_interpreted_summary from ..entity_search.models import EntityCandidate, EntityResolution, SearchEntityType +from ..observability import Phase, RequestTimings, current_request_id, current_timings from ..entity_search.resolver import DatabaseEntityResolver from ..query_understanding.context import SearchContext from ..query_understanding.service import QueryUnderstandingService @@ -80,6 +81,15 @@ class ConversationalSearchResult(BaseModel): entity_resolution_latency_ms: float = Field(default=0, alias="entityResolutionLatencyMs") db_latency_ms: float = Field(default=0, alias="dbLatencyMs") total_latency_ms: float = Field(default=0, alias="totalLatencyMs") + # Correlation id for this request, carried end to end as the X-Request-Id + # header. Named `traceId` on the wire to stay distinct from the existing + # integer `requestId`, which is the client's session generation counter and + # serves a different purpose (stale-response rejection). + # + # The full phase breakdown lives in structured logs keyed by the same id, + # and is echoed into `timings` only when EXPOSE_DEBUG_TIMINGS is on. + request_id: str | None = Field(default=None, alias="traceId") + timings: dict[str, Any] | None = None @property def is_success(self) -> bool: @@ -90,6 +100,23 @@ def total_count(self) -> int: return self.search_response.total_count if self.search_response else 0 +def record_result_attributes( + timings: RequestTimings, result: "ConversationalSearchResult" +) -> None: + """Adds the outcome attributes of one turn to the timing record. + + Called from both the service and the endpoint: the service so a directly + obtained debug snapshot is complete, the endpoint so the emitted log line + is complete even when the service is substituted. + """ + timings.update( + intent=str(result.search_plan.intent) if result.search_plan else None, + clarification=result.requires_clarification, + error_code=result.error_code, + result_count=result.total_count, + ) + + class ConversationalSearchService: """Orchestrates multi-turn conversational search: @@ -364,8 +391,42 @@ async def search( session: SearchConversationSession | None = None, language: str | None = None, current_year: int = 2026, + expose_timings: bool = False, + ) -> tuple[SearchConversationSession, ConversationalSearchResult]: + """Runs one conversational turn and stamps it with request identity. + + Every early return inside `_search` is a legitimate outcome + (clarification, no-match, parse failure), and each one is worth timing. + Stamping here rather than at each return site keeps the ~12 exit paths + from drifting apart. + """ + timings = current_timings() + updated_session, result = await self._search( + natural_query, + session=session, + language=language, + current_year=current_year, + ) + request_id = current_request_id() + snapshot: dict[str, Any] | None = None + if timings is not None: + record_result_attributes(timings, result) + if expose_timings: + snapshot = timings.snapshot() + return updated_session, result.model_copy( + update={"request_id": request_id, "timings": snapshot} + ) + + async def _search( + self, + natural_query: str, + *, + session: SearchConversationSession | None = None, + language: str | None = None, + current_year: int = 2026, ) -> tuple[SearchConversationSession, ConversationalSearchResult]: current_session = session or SearchConversationSession() + timings = current_timings() started = time.perf_counter() clean = natural_query.strip() @@ -411,6 +472,16 @@ async def search( context=search_context, ) parse_ms = (time.perf_counter() - parse_start) * 1000 + if timings is not None: + timings.add(Phase.QUERY_UNDERSTANDING, parse_ms) + timings.update( + query_understanding_source="gemini", + gemini_called=True, + model=parse_result.model, + provider=parse_result.provider, + provider_attempts=parse_result.attempts, + provider_retries=parse_result.provider_retries, + ) if parse_result.requires_clarification: elapsed = (time.perf_counter() - started) * 1000 @@ -435,6 +506,7 @@ async def search( ) return request_session, result + recovery_start = time.perf_counter() parsed_query, neutralized_temporal_filters = self._neutralize_ungrounded_temporal_filters( parse_result.query, clean, @@ -453,6 +525,10 @@ async def search( parsed_query = self._apply_referent_fallback( parsed_query, request_session.referents ) + if timings is not None: + timings.add( + Phase.DETERMINISTIC_RECOVERY, (time.perf_counter() - recovery_start) * 1000 + ) missing_subject_question = self._missing_required_subject(parsed_query) if missing_subject_question is not None: elapsed = (time.perf_counter() - started) * 1000 @@ -527,6 +603,8 @@ async def search( context=search_context, ) er_ms = (time.perf_counter() - er_start) * 1000 + if timings is not None: + timings.add(Phase.ENTITY_RESOLUTION, er_ms) candidates = resolution_result.candidates resolutions = resolution_result.resolutions @@ -569,12 +647,15 @@ async def search( }) # Step 4: Deterministic SearchPlan Compilation & Validation + plan_start = time.perf_counter() try: search_plan = self.plan_builder.build(resolved_query, resolutions=resolutions) if trusted_rally_ids: search_plan = search_plan.model_copy(update={ "event_ids": [*trusted_rally_ids, *search_plan.event_ids], }) + if timings is not None: + timings.add(Phase.SEARCH_PLAN, (time.perf_counter() - plan_start) * 1000) except UnresolvedEntityError as exc: elapsed = (time.perf_counter() - started) * 1000 result = ConversationalSearchResult( @@ -618,6 +699,8 @@ async def search( try: search_response = await self.repository.search(search_plan) db_ms = (time.perf_counter() - db_start) * 1000 + if timings is not None: + timings.add(Phase.REPOSITORY_DB, db_ms) except Exception as exc: elapsed = (time.perf_counter() - started) * 1000 result = ConversationalSearchResult( diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-09-20-544181Z/connectivity_fallback_results.json b/backend/benchmarks/results/offline_search_2026-09-03T20-09-20-544181Z/connectivity_fallback_results.json new file mode 100644 index 0000000..e44473f --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-09-20-544181Z/connectivity_fallback_results.json @@ -0,0 +1,26 @@ +[ + { + "scenario": "ONLINE", + "mode": "onlineAuthoritative", + "ux_state": "online", + "silent_swap": false + }, + { + "scenario": "OFFLINE", + "mode": "offlineLocal", + "ux_state": "offlineLocalResults", + "silent_swap": false + }, + { + "scenario": "TIMEOUT", + "mode": "lowBandwidthLocal", + "ux_state": "lowBandwidthLocalFallback", + "silent_swap": false + }, + { + "scenario": "BACKEND_ERROR", + "mode": "backendUnreachableLocal", + "ux_state": "backendUnreachableLocalAvailable", + "silent_swap": false + } +] \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-09-20-544181Z/offline_benchmark_metadata.json b/backend/benchmarks/results/offline_search_2026-09-03T20-09-20-544181Z/offline_benchmark_metadata.json new file mode 100644 index 0000000..38fc474 --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-09-20-544181Z/offline_benchmark_metadata.json @@ -0,0 +1,14 @@ +{ + "generated_at": "2026-09-03T20-09-20-544181Z", + "corpus_size": 41, + "intent_accuracy": 1.0, + "field_f1": 1.0, + "entity_accuracy": 1.0, + "clarification_accuracy": 1.0, + "safe_unsupported_rate": 1.0, + "special_accuracy": 1.0, + "wrong_confident": 0, + "offline_coverage_rate": 0.8888888888888888, + "execution_parity_pass": 16, + "execution_parity_total": 16 +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-09-20-544181Z/offline_execution_parity.jsonl b/backend/benchmarks/results/offline_search_2026-09-03T20-09-20-544181Z/offline_execution_parity.jsonl new file mode 100644 index 0000000..d5ce7ce --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-09-20-544181Z/offline_execution_parity.jsonl @@ -0,0 +1,16 @@ +{"name":"rallies_ireland_2025","intent":"SEARCH_RALLIES","online_total":24,"offline_total":24,"match":true} +{"name":"rallies_ireland_all","intent":"SEARCH_RALLIES","online_total":35,"offline_total":35,"match":true} +{"name":"rallies_portugal","intent":"SEARCH_RALLIES","online_total":7,"offline_total":7,"match":true} +{"name":"rallies_year_2026","intent":"SEARCH_RALLIES","online_total":54,"offline_total":54,"match":true} +{"name":"rally_aluksne_byid","intent":"SEARCH_RALLIES","online_total":1,"offline_total":1,"match":true} +{"name":"driver_rallies_freeman","intent":"SEARCH_DRIVER_RALLIES","online_total":9,"offline_total":9,"match":true} +{"name":"driver_wins_moffett","intent":"SEARCH_DRIVER_WINS","online_total":1,"offline_total":1,"match":true} +{"name":"rally_results_aluksne","intent":"GET_RALLY_RESULTS","online_total":1,"offline_total":1,"match":true} +{"name":"rally_results_donegal","intent":"GET_RALLY_RESULTS","online_total":1,"offline_total":1,"match":true} +{"name":"top_finishers_aluksne","intent":"GET_RALLY_TOP_FINISHERS","online_total":81,"offline_total":81,"match":true} +{"name":"top_finishers_donegal","intent":"GET_RALLY_TOP_FINISHERS","online_total":164,"offline_total":164,"match":true} +{"name":"top_uploaders_global","intent":"GET_TOP_UPLOADERS","online_total":259,"offline_total":259,"match":true} +{"name":"top_drivers_by_wins","intent":"GET_TOP_DRIVERS_BY_WINS","online_total":5,"offline_total":5,"match":true} +{"name":"driver_videos_freeman","intent":"SEARCH_DRIVER_VIDEOS","online_total":53,"offline_total":53,"match":true} +{"name":"video_actions_jumps","intent":"SEARCH_VIDEO_ACTIONS","online_total":3077,"offline_total":3077,"match":true} +{"name":"video_actions_crashes","intent":"SEARCH_VIDEO_ACTIONS","online_total":1627,"offline_total":1627,"match":true} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-09-20-544181Z/offline_failure_analysis.json b/backend/benchmarks/results/offline_search_2026-09-03T20-09-20-544181Z/offline_failure_analysis.json new file mode 100644 index 0000000..db8a271 --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-09-20-544181Z/offline_failure_analysis.json @@ -0,0 +1,4 @@ +{ + "wrong_confident": 0, + "failures": [] +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-09-20-544181Z/offline_parser_results.jsonl b/backend/benchmarks/results/offline_search_2026-09-03T20-09-20-544181Z/offline_parser_results.jsonl new file mode 100644 index 0000000..1308dba --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-09-20-544181Z/offline_parser_results.jsonl @@ -0,0 +1,41 @@ +{"query":"rallies in ireland in 2025","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","countries":["ireland"],"country":"ireland","years":[2025],"year":2025,"driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"what rallies happened in ireland","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","countries":["ireland"],"country":"ireland","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"portugal rallies 2024","category":"multi_filter","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","countries":["portugal"],"country":"portugal","years":[2024],"year":2024,"driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rally aluksne","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rally aluqsne","category":"typo","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rallies between 2024 and 2026","category":"multi_filter","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","yearFrom":2024,"yearTo":2026,"driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"max freeman rallies","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_RALLIES","actual_intent":"SEARCH_DRIVER_RALLIES","actual_fields":{"intent":"SEARCH_DRIVER_RALLIES","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"what rallies did max freeman compete in","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_RALLIES","actual_intent":"SEARCH_DRIVER_RALLIES","actual_fields":{"intent":"SEARCH_DRIVER_RALLIES","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"max freemn rallies","category":"typo","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_RALLIES","actual_intent":"SEARCH_DRIVER_RALLIES","actual_fields":{"intent":"SEARCH_DRIVER_RALLIES","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rallies won by max freeman","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_WINS","actual_intent":"SEARCH_DRIVER_WINS","actual_fields":{"intent":"SEARCH_DRIVER_WINS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"max freeman wins","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_WINS","actual_intent":"SEARCH_DRIVER_WINS","actual_fields":{"intent":"SEARCH_DRIVER_WINS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"who won rally aluksne","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_RESULTS","actual_intent":"GET_RALLY_RESULTS","actual_fields":{"intent":"GET_RALLY_RESULTS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"winner of rally aluksne","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_RESULTS","actual_intent":"GET_RALLY_RESULTS","actual_fields":{"intent":"GET_RALLY_RESULTS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"results for rally aluksne","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_TOP_FINISHERS","actual_intent":"GET_RALLY_TOP_FINISHERS","actual_fields":{"intent":"GET_RALLY_TOP_FINISHERS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"leaderboard for rally aluksne","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_TOP_FINISHERS","actual_intent":"GET_RALLY_TOP_FINISHERS","actual_fields":{"intent":"GET_RALLY_TOP_FINISHERS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"jumps from rally ireland","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_VIDEO_ACTIONS","actual_intent":"SEARCH_VIDEO_ACTIONS","actual_fields":{"intent":"SEARCH_VIDEO_ACTIONS","countries":["ireland"],"country":"ireland","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"show me crashes","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_VIDEO_ACTIONS","actual_intent":"SEARCH_VIDEO_ACTIONS","actual_fields":{"intent":"SEARCH_VIDEO_ACTIONS","actionTypes":["crash"],"actionType":"crash","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"drifts of max freeman","category":"multi_filter","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_VIDEO_ACTIONS","actual_intent":"SEARCH_VIDEO_ACTIONS","actual_fields":{"intent":"SEARCH_VIDEO_ACTIONS","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","actionTypes":["drift"],"actionType":"drift","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"videos of max freeman","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_VIDEOS","actual_intent":"SEARCH_DRIVER_VIDEOS","actual_fields":{"intent":"SEARCH_DRIVER_VIDEOS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"watch max freeman onboard","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_VIDEOS","actual_intent":"SEARCH_DRIVER_VIDEOS","actual_fields":{"intent":"SEARCH_DRIVER_VIDEOS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"top uploaders","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_UPLOADERS","actual_intent":"GET_TOP_UPLOADERS","actual_fields":{"intent":"GET_TOP_UPLOADERS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"who uploaded the most videos","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_UPLOADERS","actual_intent":"GET_TOP_UPLOADERS","actual_fields":{"intent":"GET_TOP_UPLOADERS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"top drivers by wins","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_DRIVERS_BY_WINS","actual_intent":"GET_TOP_DRIVERS_BY_WINS","actual_fields":{"intent":"GET_TOP_DRIVERS_BY_WINS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"most successful drivers","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_DRIVERS_BY_WINS","actual_intent":"GET_TOP_DRIVERS_BY_WINS","actual_fields":{"intent":"GET_TOP_DRIVERS_BY_WINS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rally donegal","category":"ambiguity","expected_kind":"clarification","actual_kind":"clarification","expected_intent":null,"actual_intent":"SEARCH_RALLIES","actual_fields":null} +{"query":"who won rally donegal","category":"ambiguity","expected_kind":"clarification","actual_kind":"clarification","expected_intent":null,"actual_intent":"GET_RALLY_RESULTS","actual_fields":null} +{"query":"results for rally donegal","category":"ambiguity","expected_kind":"clarification","actual_kind":"clarification","expected_intent":null,"actual_intent":"GET_RALLY_TOP_FINISHERS","actual_fields":null} +{"query":"explain the offside rule in football","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what is the meaning of life","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"book me a flight to tokyo","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"rally zzzxqywv","category":"unsupported","expected_kind":"noMatch","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"compare aerodynamics of wrc cars in detail","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what is the weather","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"hello","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"thanks","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"who are you","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what can you do","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"tell me a joke","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"are you alive","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"who is the best rally driver of all time","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what is the capital of france","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-09-20-544181Z/offline_search_report.md b/backend/benchmarks/results/offline_search_2026-09-03T20-09-20-544181Z/offline_search_report.md new file mode 100644 index 0000000..3ed2a12 --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-09-20-544181Z/offline_search_report.md @@ -0,0 +1,29 @@ +# Offline Search Benchmark Report + +Generated: 2026-09-03T20-09-20-544181Z + +## Primary safety gate +- **wrong_confident: 0** (gate: must be 0) + +## Parser metrics +- Corpus size: 41 +- Intent accuracy: 100.0% (24/24) +- Field F1: 1.000 +- Entity resolution accuracy: 100.0% (14/14) +- Clarification accuracy: 100.0% (3/3) +- Safe unsupported rate: 100.0% (5/5) +- Special-query accuracy: 100.0% (9/9) +- OFFLINE_COVERAGE_RATE: 88.9% (24/27 answerable produced results) + +## Execution parity (offline SQLite vs online MySQL oracle) +- Cases matched: 16/16 + +## Connectivity fallback +- ONLINE: onlineAuthoritative +- OFFLINE: offlineLocal +- TIMEOUT: lowBandwidthLocal +- BACKEND_ERROR: backendUnreachableLocal + +## Voice offline +- Cloud voice offline: NO +- On-device voice offline: DEVICE_DEPENDENT diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-09-20-544181Z/snapshot_validation.json b/backend/benchmarks/results/offline_search_2026-09-03T20-09-20-544181Z/snapshot_validation.json new file mode 100644 index 0000000..0365224 --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-09-20-544181Z/snapshot_validation.json @@ -0,0 +1,14 @@ +{ + "snapshot_id": "1-4b19d6e0547dcab0-full", + "table_counts": { + "rallies": 111, + "people": 8750, + "stages": 1025, + "participation": 9967, + "final_results": 326, + "driver_wins": 5, + "uploader_stats": 259, + "video_meta": 18510, + "video_actions": 32497 + } +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-09-20-544181Z/special_query_results.json b/backend/benchmarks/results/offline_search_2026-09-03T20-09-20-544181Z/special_query_results.json new file mode 100644 index 0000000..279124a --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-09-20-544181Z/special_query_results.json @@ -0,0 +1,47 @@ +[ + { + "query": "what is the weather", + "category": "weather", + "intercepted": true + }, + { + "query": "hello", + "category": "greeting", + "intercepted": true + }, + { + "query": "thanks", + "category": "thanks", + "intercepted": true + }, + { + "query": "who are you", + "category": "identity", + "intercepted": true + }, + { + "query": "what can you do", + "category": "capabilities", + "intercepted": true + }, + { + "query": "tell me a joke", + "category": "joke", + "intercepted": true + }, + { + "query": "are you alive", + "category": "alive", + "intercepted": true + }, + { + "query": "who is the best rally driver of all time", + "category": "rallyOpinion", + "intercepted": true + }, + { + "query": "what is the capital of france", + "category": "unsupported", + "intercepted": true + } +] \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-09-20-544181Z/voice_offline_results.json b/backend/benchmarks/results/offline_search_2026-09-03T20-09-20-544181Z/voice_offline_results.json new file mode 100644 index 0000000..b99b8cd --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-09-20-544181Z/voice_offline_results.json @@ -0,0 +1,6 @@ +{ + "cloud_voice_offline": "NO (network required)", + "on_device_voice_offline": "DEVICE_DEPENDENT (only where OS on-device recognizer supports it)", + "auto_submit": false, + "transcript_editable": true +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-10-34-805077Z/connectivity_fallback_results.json b/backend/benchmarks/results/offline_search_2026-09-03T20-10-34-805077Z/connectivity_fallback_results.json new file mode 100644 index 0000000..e44473f --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-10-34-805077Z/connectivity_fallback_results.json @@ -0,0 +1,26 @@ +[ + { + "scenario": "ONLINE", + "mode": "onlineAuthoritative", + "ux_state": "online", + "silent_swap": false + }, + { + "scenario": "OFFLINE", + "mode": "offlineLocal", + "ux_state": "offlineLocalResults", + "silent_swap": false + }, + { + "scenario": "TIMEOUT", + "mode": "lowBandwidthLocal", + "ux_state": "lowBandwidthLocalFallback", + "silent_swap": false + }, + { + "scenario": "BACKEND_ERROR", + "mode": "backendUnreachableLocal", + "ux_state": "backendUnreachableLocalAvailable", + "silent_swap": false + } +] \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-10-34-805077Z/offline_benchmark_metadata.json b/backend/benchmarks/results/offline_search_2026-09-03T20-10-34-805077Z/offline_benchmark_metadata.json new file mode 100644 index 0000000..420fbfc --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-10-34-805077Z/offline_benchmark_metadata.json @@ -0,0 +1,14 @@ +{ + "generated_at": "2026-09-03T20-10-34-805077Z", + "corpus_size": 41, + "intent_accuracy": 1.0, + "field_f1": 1.0, + "entity_accuracy": 1.0, + "clarification_accuracy": 1.0, + "safe_unsupported_rate": 1.0, + "special_accuracy": 1.0, + "wrong_confident": 0, + "offline_coverage_rate": 0.8888888888888888, + "execution_parity_pass": 16, + "execution_parity_total": 16 +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-10-34-805077Z/offline_execution_parity.jsonl b/backend/benchmarks/results/offline_search_2026-09-03T20-10-34-805077Z/offline_execution_parity.jsonl new file mode 100644 index 0000000..d5ce7ce --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-10-34-805077Z/offline_execution_parity.jsonl @@ -0,0 +1,16 @@ +{"name":"rallies_ireland_2025","intent":"SEARCH_RALLIES","online_total":24,"offline_total":24,"match":true} +{"name":"rallies_ireland_all","intent":"SEARCH_RALLIES","online_total":35,"offline_total":35,"match":true} +{"name":"rallies_portugal","intent":"SEARCH_RALLIES","online_total":7,"offline_total":7,"match":true} +{"name":"rallies_year_2026","intent":"SEARCH_RALLIES","online_total":54,"offline_total":54,"match":true} +{"name":"rally_aluksne_byid","intent":"SEARCH_RALLIES","online_total":1,"offline_total":1,"match":true} +{"name":"driver_rallies_freeman","intent":"SEARCH_DRIVER_RALLIES","online_total":9,"offline_total":9,"match":true} +{"name":"driver_wins_moffett","intent":"SEARCH_DRIVER_WINS","online_total":1,"offline_total":1,"match":true} +{"name":"rally_results_aluksne","intent":"GET_RALLY_RESULTS","online_total":1,"offline_total":1,"match":true} +{"name":"rally_results_donegal","intent":"GET_RALLY_RESULTS","online_total":1,"offline_total":1,"match":true} +{"name":"top_finishers_aluksne","intent":"GET_RALLY_TOP_FINISHERS","online_total":81,"offline_total":81,"match":true} +{"name":"top_finishers_donegal","intent":"GET_RALLY_TOP_FINISHERS","online_total":164,"offline_total":164,"match":true} +{"name":"top_uploaders_global","intent":"GET_TOP_UPLOADERS","online_total":259,"offline_total":259,"match":true} +{"name":"top_drivers_by_wins","intent":"GET_TOP_DRIVERS_BY_WINS","online_total":5,"offline_total":5,"match":true} +{"name":"driver_videos_freeman","intent":"SEARCH_DRIVER_VIDEOS","online_total":53,"offline_total":53,"match":true} +{"name":"video_actions_jumps","intent":"SEARCH_VIDEO_ACTIONS","online_total":3077,"offline_total":3077,"match":true} +{"name":"video_actions_crashes","intent":"SEARCH_VIDEO_ACTIONS","online_total":1627,"offline_total":1627,"match":true} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-10-34-805077Z/offline_failure_analysis.json b/backend/benchmarks/results/offline_search_2026-09-03T20-10-34-805077Z/offline_failure_analysis.json new file mode 100644 index 0000000..db8a271 --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-10-34-805077Z/offline_failure_analysis.json @@ -0,0 +1,4 @@ +{ + "wrong_confident": 0, + "failures": [] +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-10-34-805077Z/offline_parser_results.jsonl b/backend/benchmarks/results/offline_search_2026-09-03T20-10-34-805077Z/offline_parser_results.jsonl new file mode 100644 index 0000000..1308dba --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-10-34-805077Z/offline_parser_results.jsonl @@ -0,0 +1,41 @@ +{"query":"rallies in ireland in 2025","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","countries":["ireland"],"country":"ireland","years":[2025],"year":2025,"driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"what rallies happened in ireland","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","countries":["ireland"],"country":"ireland","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"portugal rallies 2024","category":"multi_filter","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","countries":["portugal"],"country":"portugal","years":[2024],"year":2024,"driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rally aluksne","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rally aluqsne","category":"typo","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rallies between 2024 and 2026","category":"multi_filter","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","yearFrom":2024,"yearTo":2026,"driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"max freeman rallies","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_RALLIES","actual_intent":"SEARCH_DRIVER_RALLIES","actual_fields":{"intent":"SEARCH_DRIVER_RALLIES","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"what rallies did max freeman compete in","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_RALLIES","actual_intent":"SEARCH_DRIVER_RALLIES","actual_fields":{"intent":"SEARCH_DRIVER_RALLIES","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"max freemn rallies","category":"typo","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_RALLIES","actual_intent":"SEARCH_DRIVER_RALLIES","actual_fields":{"intent":"SEARCH_DRIVER_RALLIES","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rallies won by max freeman","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_WINS","actual_intent":"SEARCH_DRIVER_WINS","actual_fields":{"intent":"SEARCH_DRIVER_WINS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"max freeman wins","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_WINS","actual_intent":"SEARCH_DRIVER_WINS","actual_fields":{"intent":"SEARCH_DRIVER_WINS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"who won rally aluksne","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_RESULTS","actual_intent":"GET_RALLY_RESULTS","actual_fields":{"intent":"GET_RALLY_RESULTS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"winner of rally aluksne","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_RESULTS","actual_intent":"GET_RALLY_RESULTS","actual_fields":{"intent":"GET_RALLY_RESULTS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"results for rally aluksne","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_TOP_FINISHERS","actual_intent":"GET_RALLY_TOP_FINISHERS","actual_fields":{"intent":"GET_RALLY_TOP_FINISHERS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"leaderboard for rally aluksne","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_TOP_FINISHERS","actual_intent":"GET_RALLY_TOP_FINISHERS","actual_fields":{"intent":"GET_RALLY_TOP_FINISHERS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"jumps from rally ireland","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_VIDEO_ACTIONS","actual_intent":"SEARCH_VIDEO_ACTIONS","actual_fields":{"intent":"SEARCH_VIDEO_ACTIONS","countries":["ireland"],"country":"ireland","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"show me crashes","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_VIDEO_ACTIONS","actual_intent":"SEARCH_VIDEO_ACTIONS","actual_fields":{"intent":"SEARCH_VIDEO_ACTIONS","actionTypes":["crash"],"actionType":"crash","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"drifts of max freeman","category":"multi_filter","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_VIDEO_ACTIONS","actual_intent":"SEARCH_VIDEO_ACTIONS","actual_fields":{"intent":"SEARCH_VIDEO_ACTIONS","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","actionTypes":["drift"],"actionType":"drift","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"videos of max freeman","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_VIDEOS","actual_intent":"SEARCH_DRIVER_VIDEOS","actual_fields":{"intent":"SEARCH_DRIVER_VIDEOS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"watch max freeman onboard","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_VIDEOS","actual_intent":"SEARCH_DRIVER_VIDEOS","actual_fields":{"intent":"SEARCH_DRIVER_VIDEOS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"top uploaders","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_UPLOADERS","actual_intent":"GET_TOP_UPLOADERS","actual_fields":{"intent":"GET_TOP_UPLOADERS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"who uploaded the most videos","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_UPLOADERS","actual_intent":"GET_TOP_UPLOADERS","actual_fields":{"intent":"GET_TOP_UPLOADERS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"top drivers by wins","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_DRIVERS_BY_WINS","actual_intent":"GET_TOP_DRIVERS_BY_WINS","actual_fields":{"intent":"GET_TOP_DRIVERS_BY_WINS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"most successful drivers","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_DRIVERS_BY_WINS","actual_intent":"GET_TOP_DRIVERS_BY_WINS","actual_fields":{"intent":"GET_TOP_DRIVERS_BY_WINS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rally donegal","category":"ambiguity","expected_kind":"clarification","actual_kind":"clarification","expected_intent":null,"actual_intent":"SEARCH_RALLIES","actual_fields":null} +{"query":"who won rally donegal","category":"ambiguity","expected_kind":"clarification","actual_kind":"clarification","expected_intent":null,"actual_intent":"GET_RALLY_RESULTS","actual_fields":null} +{"query":"results for rally donegal","category":"ambiguity","expected_kind":"clarification","actual_kind":"clarification","expected_intent":null,"actual_intent":"GET_RALLY_TOP_FINISHERS","actual_fields":null} +{"query":"explain the offside rule in football","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what is the meaning of life","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"book me a flight to tokyo","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"rally zzzxqywv","category":"unsupported","expected_kind":"noMatch","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"compare aerodynamics of wrc cars in detail","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what is the weather","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"hello","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"thanks","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"who are you","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what can you do","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"tell me a joke","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"are you alive","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"who is the best rally driver of all time","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what is the capital of france","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-10-34-805077Z/offline_search_report.md b/backend/benchmarks/results/offline_search_2026-09-03T20-10-34-805077Z/offline_search_report.md new file mode 100644 index 0000000..4ce482a --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-10-34-805077Z/offline_search_report.md @@ -0,0 +1,29 @@ +# Offline Search Benchmark Report + +Generated: 2026-09-03T20-10-34-805077Z + +## Primary safety gate +- **wrong_confident: 0** (gate: must be 0) + +## Parser metrics +- Corpus size: 41 +- Intent accuracy: 100.0% (24/24) +- Field F1: 1.000 +- Entity resolution accuracy: 100.0% (14/14) +- Clarification accuracy: 100.0% (3/3) +- Safe unsupported rate: 100.0% (5/5) +- Special-query accuracy: 100.0% (9/9) +- OFFLINE_COVERAGE_RATE: 88.9% (24/27 answerable produced results) + +## Execution parity (offline SQLite vs online MySQL oracle) +- Cases matched: 16/16 + +## Connectivity fallback +- ONLINE: onlineAuthoritative +- OFFLINE: offlineLocal +- TIMEOUT: lowBandwidthLocal +- BACKEND_ERROR: backendUnreachableLocal + +## Voice offline +- Cloud voice offline: NO +- On-device voice offline: DEVICE_DEPENDENT diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-10-34-805077Z/snapshot_validation.json b/backend/benchmarks/results/offline_search_2026-09-03T20-10-34-805077Z/snapshot_validation.json new file mode 100644 index 0000000..0365224 --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-10-34-805077Z/snapshot_validation.json @@ -0,0 +1,14 @@ +{ + "snapshot_id": "1-4b19d6e0547dcab0-full", + "table_counts": { + "rallies": 111, + "people": 8750, + "stages": 1025, + "participation": 9967, + "final_results": 326, + "driver_wins": 5, + "uploader_stats": 259, + "video_meta": 18510, + "video_actions": 32497 + } +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-10-34-805077Z/special_query_results.json b/backend/benchmarks/results/offline_search_2026-09-03T20-10-34-805077Z/special_query_results.json new file mode 100644 index 0000000..279124a --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-10-34-805077Z/special_query_results.json @@ -0,0 +1,47 @@ +[ + { + "query": "what is the weather", + "category": "weather", + "intercepted": true + }, + { + "query": "hello", + "category": "greeting", + "intercepted": true + }, + { + "query": "thanks", + "category": "thanks", + "intercepted": true + }, + { + "query": "who are you", + "category": "identity", + "intercepted": true + }, + { + "query": "what can you do", + "category": "capabilities", + "intercepted": true + }, + { + "query": "tell me a joke", + "category": "joke", + "intercepted": true + }, + { + "query": "are you alive", + "category": "alive", + "intercepted": true + }, + { + "query": "who is the best rally driver of all time", + "category": "rallyOpinion", + "intercepted": true + }, + { + "query": "what is the capital of france", + "category": "unsupported", + "intercepted": true + } +] \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-10-34-805077Z/voice_offline_results.json b/backend/benchmarks/results/offline_search_2026-09-03T20-10-34-805077Z/voice_offline_results.json new file mode 100644 index 0000000..b99b8cd --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-10-34-805077Z/voice_offline_results.json @@ -0,0 +1,6 @@ +{ + "cloud_voice_offline": "NO (network required)", + "on_device_voice_offline": "DEVICE_DEPENDENT (only where OS on-device recognizer supports it)", + "auto_submit": false, + "transcript_editable": true +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-12-12-529188Z/connectivity_fallback_results.json b/backend/benchmarks/results/offline_search_2026-09-03T20-12-12-529188Z/connectivity_fallback_results.json new file mode 100644 index 0000000..e44473f --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-12-12-529188Z/connectivity_fallback_results.json @@ -0,0 +1,26 @@ +[ + { + "scenario": "ONLINE", + "mode": "onlineAuthoritative", + "ux_state": "online", + "silent_swap": false + }, + { + "scenario": "OFFLINE", + "mode": "offlineLocal", + "ux_state": "offlineLocalResults", + "silent_swap": false + }, + { + "scenario": "TIMEOUT", + "mode": "lowBandwidthLocal", + "ux_state": "lowBandwidthLocalFallback", + "silent_swap": false + }, + { + "scenario": "BACKEND_ERROR", + "mode": "backendUnreachableLocal", + "ux_state": "backendUnreachableLocalAvailable", + "silent_swap": false + } +] \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-12-12-529188Z/offline_benchmark_metadata.json b/backend/benchmarks/results/offline_search_2026-09-03T20-12-12-529188Z/offline_benchmark_metadata.json new file mode 100644 index 0000000..4b446bb --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-12-12-529188Z/offline_benchmark_metadata.json @@ -0,0 +1,14 @@ +{ + "generated_at": "2026-09-03T20-12-12-529188Z", + "corpus_size": 41, + "intent_accuracy": 1.0, + "field_f1": 1.0, + "entity_accuracy": 1.0, + "clarification_accuracy": 1.0, + "safe_unsupported_rate": 1.0, + "special_accuracy": 1.0, + "wrong_confident": 0, + "offline_coverage_rate": 0.8888888888888888, + "execution_parity_pass": 16, + "execution_parity_total": 16 +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-12-12-529188Z/offline_execution_parity.jsonl b/backend/benchmarks/results/offline_search_2026-09-03T20-12-12-529188Z/offline_execution_parity.jsonl new file mode 100644 index 0000000..d5ce7ce --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-12-12-529188Z/offline_execution_parity.jsonl @@ -0,0 +1,16 @@ +{"name":"rallies_ireland_2025","intent":"SEARCH_RALLIES","online_total":24,"offline_total":24,"match":true} +{"name":"rallies_ireland_all","intent":"SEARCH_RALLIES","online_total":35,"offline_total":35,"match":true} +{"name":"rallies_portugal","intent":"SEARCH_RALLIES","online_total":7,"offline_total":7,"match":true} +{"name":"rallies_year_2026","intent":"SEARCH_RALLIES","online_total":54,"offline_total":54,"match":true} +{"name":"rally_aluksne_byid","intent":"SEARCH_RALLIES","online_total":1,"offline_total":1,"match":true} +{"name":"driver_rallies_freeman","intent":"SEARCH_DRIVER_RALLIES","online_total":9,"offline_total":9,"match":true} +{"name":"driver_wins_moffett","intent":"SEARCH_DRIVER_WINS","online_total":1,"offline_total":1,"match":true} +{"name":"rally_results_aluksne","intent":"GET_RALLY_RESULTS","online_total":1,"offline_total":1,"match":true} +{"name":"rally_results_donegal","intent":"GET_RALLY_RESULTS","online_total":1,"offline_total":1,"match":true} +{"name":"top_finishers_aluksne","intent":"GET_RALLY_TOP_FINISHERS","online_total":81,"offline_total":81,"match":true} +{"name":"top_finishers_donegal","intent":"GET_RALLY_TOP_FINISHERS","online_total":164,"offline_total":164,"match":true} +{"name":"top_uploaders_global","intent":"GET_TOP_UPLOADERS","online_total":259,"offline_total":259,"match":true} +{"name":"top_drivers_by_wins","intent":"GET_TOP_DRIVERS_BY_WINS","online_total":5,"offline_total":5,"match":true} +{"name":"driver_videos_freeman","intent":"SEARCH_DRIVER_VIDEOS","online_total":53,"offline_total":53,"match":true} +{"name":"video_actions_jumps","intent":"SEARCH_VIDEO_ACTIONS","online_total":3077,"offline_total":3077,"match":true} +{"name":"video_actions_crashes","intent":"SEARCH_VIDEO_ACTIONS","online_total":1627,"offline_total":1627,"match":true} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-12-12-529188Z/offline_failure_analysis.json b/backend/benchmarks/results/offline_search_2026-09-03T20-12-12-529188Z/offline_failure_analysis.json new file mode 100644 index 0000000..db8a271 --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-12-12-529188Z/offline_failure_analysis.json @@ -0,0 +1,4 @@ +{ + "wrong_confident": 0, + "failures": [] +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-12-12-529188Z/offline_parser_results.jsonl b/backend/benchmarks/results/offline_search_2026-09-03T20-12-12-529188Z/offline_parser_results.jsonl new file mode 100644 index 0000000..1308dba --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-12-12-529188Z/offline_parser_results.jsonl @@ -0,0 +1,41 @@ +{"query":"rallies in ireland in 2025","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","countries":["ireland"],"country":"ireland","years":[2025],"year":2025,"driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"what rallies happened in ireland","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","countries":["ireland"],"country":"ireland","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"portugal rallies 2024","category":"multi_filter","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","countries":["portugal"],"country":"portugal","years":[2024],"year":2024,"driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rally aluksne","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rally aluqsne","category":"typo","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rallies between 2024 and 2026","category":"multi_filter","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","yearFrom":2024,"yearTo":2026,"driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"max freeman rallies","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_RALLIES","actual_intent":"SEARCH_DRIVER_RALLIES","actual_fields":{"intent":"SEARCH_DRIVER_RALLIES","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"what rallies did max freeman compete in","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_RALLIES","actual_intent":"SEARCH_DRIVER_RALLIES","actual_fields":{"intent":"SEARCH_DRIVER_RALLIES","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"max freemn rallies","category":"typo","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_RALLIES","actual_intent":"SEARCH_DRIVER_RALLIES","actual_fields":{"intent":"SEARCH_DRIVER_RALLIES","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rallies won by max freeman","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_WINS","actual_intent":"SEARCH_DRIVER_WINS","actual_fields":{"intent":"SEARCH_DRIVER_WINS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"max freeman wins","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_WINS","actual_intent":"SEARCH_DRIVER_WINS","actual_fields":{"intent":"SEARCH_DRIVER_WINS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"who won rally aluksne","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_RESULTS","actual_intent":"GET_RALLY_RESULTS","actual_fields":{"intent":"GET_RALLY_RESULTS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"winner of rally aluksne","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_RESULTS","actual_intent":"GET_RALLY_RESULTS","actual_fields":{"intent":"GET_RALLY_RESULTS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"results for rally aluksne","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_TOP_FINISHERS","actual_intent":"GET_RALLY_TOP_FINISHERS","actual_fields":{"intent":"GET_RALLY_TOP_FINISHERS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"leaderboard for rally aluksne","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_TOP_FINISHERS","actual_intent":"GET_RALLY_TOP_FINISHERS","actual_fields":{"intent":"GET_RALLY_TOP_FINISHERS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"jumps from rally ireland","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_VIDEO_ACTIONS","actual_intent":"SEARCH_VIDEO_ACTIONS","actual_fields":{"intent":"SEARCH_VIDEO_ACTIONS","countries":["ireland"],"country":"ireland","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"show me crashes","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_VIDEO_ACTIONS","actual_intent":"SEARCH_VIDEO_ACTIONS","actual_fields":{"intent":"SEARCH_VIDEO_ACTIONS","actionTypes":["crash"],"actionType":"crash","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"drifts of max freeman","category":"multi_filter","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_VIDEO_ACTIONS","actual_intent":"SEARCH_VIDEO_ACTIONS","actual_fields":{"intent":"SEARCH_VIDEO_ACTIONS","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","actionTypes":["drift"],"actionType":"drift","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"videos of max freeman","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_VIDEOS","actual_intent":"SEARCH_DRIVER_VIDEOS","actual_fields":{"intent":"SEARCH_DRIVER_VIDEOS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"watch max freeman onboard","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_VIDEOS","actual_intent":"SEARCH_DRIVER_VIDEOS","actual_fields":{"intent":"SEARCH_DRIVER_VIDEOS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"top uploaders","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_UPLOADERS","actual_intent":"GET_TOP_UPLOADERS","actual_fields":{"intent":"GET_TOP_UPLOADERS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"who uploaded the most videos","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_UPLOADERS","actual_intent":"GET_TOP_UPLOADERS","actual_fields":{"intent":"GET_TOP_UPLOADERS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"top drivers by wins","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_DRIVERS_BY_WINS","actual_intent":"GET_TOP_DRIVERS_BY_WINS","actual_fields":{"intent":"GET_TOP_DRIVERS_BY_WINS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"most successful drivers","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_DRIVERS_BY_WINS","actual_intent":"GET_TOP_DRIVERS_BY_WINS","actual_fields":{"intent":"GET_TOP_DRIVERS_BY_WINS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rally donegal","category":"ambiguity","expected_kind":"clarification","actual_kind":"clarification","expected_intent":null,"actual_intent":"SEARCH_RALLIES","actual_fields":null} +{"query":"who won rally donegal","category":"ambiguity","expected_kind":"clarification","actual_kind":"clarification","expected_intent":null,"actual_intent":"GET_RALLY_RESULTS","actual_fields":null} +{"query":"results for rally donegal","category":"ambiguity","expected_kind":"clarification","actual_kind":"clarification","expected_intent":null,"actual_intent":"GET_RALLY_TOP_FINISHERS","actual_fields":null} +{"query":"explain the offside rule in football","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what is the meaning of life","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"book me a flight to tokyo","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"rally zzzxqywv","category":"unsupported","expected_kind":"noMatch","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"compare aerodynamics of wrc cars in detail","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what is the weather","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"hello","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"thanks","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"who are you","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what can you do","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"tell me a joke","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"are you alive","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"who is the best rally driver of all time","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what is the capital of france","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-12-12-529188Z/offline_search_report.md b/backend/benchmarks/results/offline_search_2026-09-03T20-12-12-529188Z/offline_search_report.md new file mode 100644 index 0000000..069e601 --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-12-12-529188Z/offline_search_report.md @@ -0,0 +1,29 @@ +# Offline Search Benchmark Report + +Generated: 2026-09-03T20-12-12-529188Z + +## Primary safety gate +- **wrong_confident: 0** (gate: must be 0) + +## Parser metrics +- Corpus size: 41 +- Intent accuracy: 100.0% (24/24) +- Field F1: 1.000 +- Entity resolution accuracy: 100.0% (14/14) +- Clarification accuracy: 100.0% (3/3) +- Safe unsupported rate: 100.0% (5/5) +- Special-query accuracy: 100.0% (9/9) +- OFFLINE_COVERAGE_RATE: 88.9% (24/27 answerable produced results) + +## Execution parity (offline SQLite vs online MySQL oracle) +- Cases matched: 16/16 + +## Connectivity fallback +- ONLINE: onlineAuthoritative +- OFFLINE: offlineLocal +- TIMEOUT: lowBandwidthLocal +- BACKEND_ERROR: backendUnreachableLocal + +## Voice offline +- Cloud voice offline: NO +- On-device voice offline: DEVICE_DEPENDENT diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-12-12-529188Z/snapshot_validation.json b/backend/benchmarks/results/offline_search_2026-09-03T20-12-12-529188Z/snapshot_validation.json new file mode 100644 index 0000000..0365224 --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-12-12-529188Z/snapshot_validation.json @@ -0,0 +1,14 @@ +{ + "snapshot_id": "1-4b19d6e0547dcab0-full", + "table_counts": { + "rallies": 111, + "people": 8750, + "stages": 1025, + "participation": 9967, + "final_results": 326, + "driver_wins": 5, + "uploader_stats": 259, + "video_meta": 18510, + "video_actions": 32497 + } +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-12-12-529188Z/special_query_results.json b/backend/benchmarks/results/offline_search_2026-09-03T20-12-12-529188Z/special_query_results.json new file mode 100644 index 0000000..279124a --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-12-12-529188Z/special_query_results.json @@ -0,0 +1,47 @@ +[ + { + "query": "what is the weather", + "category": "weather", + "intercepted": true + }, + { + "query": "hello", + "category": "greeting", + "intercepted": true + }, + { + "query": "thanks", + "category": "thanks", + "intercepted": true + }, + { + "query": "who are you", + "category": "identity", + "intercepted": true + }, + { + "query": "what can you do", + "category": "capabilities", + "intercepted": true + }, + { + "query": "tell me a joke", + "category": "joke", + "intercepted": true + }, + { + "query": "are you alive", + "category": "alive", + "intercepted": true + }, + { + "query": "who is the best rally driver of all time", + "category": "rallyOpinion", + "intercepted": true + }, + { + "query": "what is the capital of france", + "category": "unsupported", + "intercepted": true + } +] \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-12-12-529188Z/voice_offline_results.json b/backend/benchmarks/results/offline_search_2026-09-03T20-12-12-529188Z/voice_offline_results.json new file mode 100644 index 0000000..b99b8cd --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-12-12-529188Z/voice_offline_results.json @@ -0,0 +1,6 @@ +{ + "cloud_voice_offline": "NO (network required)", + "on_device_voice_offline": "DEVICE_DEPENDENT (only where OS on-device recognizer supports it)", + "auto_submit": false, + "transcript_editable": true +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-13-29-244605Z/connectivity_fallback_results.json b/backend/benchmarks/results/offline_search_2026-09-03T20-13-29-244605Z/connectivity_fallback_results.json new file mode 100644 index 0000000..e44473f --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-13-29-244605Z/connectivity_fallback_results.json @@ -0,0 +1,26 @@ +[ + { + "scenario": "ONLINE", + "mode": "onlineAuthoritative", + "ux_state": "online", + "silent_swap": false + }, + { + "scenario": "OFFLINE", + "mode": "offlineLocal", + "ux_state": "offlineLocalResults", + "silent_swap": false + }, + { + "scenario": "TIMEOUT", + "mode": "lowBandwidthLocal", + "ux_state": "lowBandwidthLocalFallback", + "silent_swap": false + }, + { + "scenario": "BACKEND_ERROR", + "mode": "backendUnreachableLocal", + "ux_state": "backendUnreachableLocalAvailable", + "silent_swap": false + } +] \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-13-29-244605Z/offline_benchmark_metadata.json b/backend/benchmarks/results/offline_search_2026-09-03T20-13-29-244605Z/offline_benchmark_metadata.json new file mode 100644 index 0000000..ae9216f --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-13-29-244605Z/offline_benchmark_metadata.json @@ -0,0 +1,14 @@ +{ + "generated_at": "2026-09-03T20-13-29-244605Z", + "corpus_size": 41, + "intent_accuracy": 1.0, + "field_f1": 1.0, + "entity_accuracy": 1.0, + "clarification_accuracy": 1.0, + "safe_unsupported_rate": 1.0, + "special_accuracy": 1.0, + "wrong_confident": 0, + "offline_coverage_rate": 0.8888888888888888, + "execution_parity_pass": 16, + "execution_parity_total": 16 +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-13-29-244605Z/offline_execution_parity.jsonl b/backend/benchmarks/results/offline_search_2026-09-03T20-13-29-244605Z/offline_execution_parity.jsonl new file mode 100644 index 0000000..d5ce7ce --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-13-29-244605Z/offline_execution_parity.jsonl @@ -0,0 +1,16 @@ +{"name":"rallies_ireland_2025","intent":"SEARCH_RALLIES","online_total":24,"offline_total":24,"match":true} +{"name":"rallies_ireland_all","intent":"SEARCH_RALLIES","online_total":35,"offline_total":35,"match":true} +{"name":"rallies_portugal","intent":"SEARCH_RALLIES","online_total":7,"offline_total":7,"match":true} +{"name":"rallies_year_2026","intent":"SEARCH_RALLIES","online_total":54,"offline_total":54,"match":true} +{"name":"rally_aluksne_byid","intent":"SEARCH_RALLIES","online_total":1,"offline_total":1,"match":true} +{"name":"driver_rallies_freeman","intent":"SEARCH_DRIVER_RALLIES","online_total":9,"offline_total":9,"match":true} +{"name":"driver_wins_moffett","intent":"SEARCH_DRIVER_WINS","online_total":1,"offline_total":1,"match":true} +{"name":"rally_results_aluksne","intent":"GET_RALLY_RESULTS","online_total":1,"offline_total":1,"match":true} +{"name":"rally_results_donegal","intent":"GET_RALLY_RESULTS","online_total":1,"offline_total":1,"match":true} +{"name":"top_finishers_aluksne","intent":"GET_RALLY_TOP_FINISHERS","online_total":81,"offline_total":81,"match":true} +{"name":"top_finishers_donegal","intent":"GET_RALLY_TOP_FINISHERS","online_total":164,"offline_total":164,"match":true} +{"name":"top_uploaders_global","intent":"GET_TOP_UPLOADERS","online_total":259,"offline_total":259,"match":true} +{"name":"top_drivers_by_wins","intent":"GET_TOP_DRIVERS_BY_WINS","online_total":5,"offline_total":5,"match":true} +{"name":"driver_videos_freeman","intent":"SEARCH_DRIVER_VIDEOS","online_total":53,"offline_total":53,"match":true} +{"name":"video_actions_jumps","intent":"SEARCH_VIDEO_ACTIONS","online_total":3077,"offline_total":3077,"match":true} +{"name":"video_actions_crashes","intent":"SEARCH_VIDEO_ACTIONS","online_total":1627,"offline_total":1627,"match":true} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-13-29-244605Z/offline_failure_analysis.json b/backend/benchmarks/results/offline_search_2026-09-03T20-13-29-244605Z/offline_failure_analysis.json new file mode 100644 index 0000000..db8a271 --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-13-29-244605Z/offline_failure_analysis.json @@ -0,0 +1,4 @@ +{ + "wrong_confident": 0, + "failures": [] +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-13-29-244605Z/offline_parser_results.jsonl b/backend/benchmarks/results/offline_search_2026-09-03T20-13-29-244605Z/offline_parser_results.jsonl new file mode 100644 index 0000000..1308dba --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-13-29-244605Z/offline_parser_results.jsonl @@ -0,0 +1,41 @@ +{"query":"rallies in ireland in 2025","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","countries":["ireland"],"country":"ireland","years":[2025],"year":2025,"driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"what rallies happened in ireland","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","countries":["ireland"],"country":"ireland","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"portugal rallies 2024","category":"multi_filter","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","countries":["portugal"],"country":"portugal","years":[2024],"year":2024,"driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rally aluksne","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rally aluqsne","category":"typo","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rallies between 2024 and 2026","category":"multi_filter","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","yearFrom":2024,"yearTo":2026,"driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"max freeman rallies","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_RALLIES","actual_intent":"SEARCH_DRIVER_RALLIES","actual_fields":{"intent":"SEARCH_DRIVER_RALLIES","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"what rallies did max freeman compete in","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_RALLIES","actual_intent":"SEARCH_DRIVER_RALLIES","actual_fields":{"intent":"SEARCH_DRIVER_RALLIES","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"max freemn rallies","category":"typo","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_RALLIES","actual_intent":"SEARCH_DRIVER_RALLIES","actual_fields":{"intent":"SEARCH_DRIVER_RALLIES","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rallies won by max freeman","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_WINS","actual_intent":"SEARCH_DRIVER_WINS","actual_fields":{"intent":"SEARCH_DRIVER_WINS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"max freeman wins","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_WINS","actual_intent":"SEARCH_DRIVER_WINS","actual_fields":{"intent":"SEARCH_DRIVER_WINS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"who won rally aluksne","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_RESULTS","actual_intent":"GET_RALLY_RESULTS","actual_fields":{"intent":"GET_RALLY_RESULTS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"winner of rally aluksne","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_RESULTS","actual_intent":"GET_RALLY_RESULTS","actual_fields":{"intent":"GET_RALLY_RESULTS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"results for rally aluksne","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_TOP_FINISHERS","actual_intent":"GET_RALLY_TOP_FINISHERS","actual_fields":{"intent":"GET_RALLY_TOP_FINISHERS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"leaderboard for rally aluksne","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_TOP_FINISHERS","actual_intent":"GET_RALLY_TOP_FINISHERS","actual_fields":{"intent":"GET_RALLY_TOP_FINISHERS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"jumps from rally ireland","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_VIDEO_ACTIONS","actual_intent":"SEARCH_VIDEO_ACTIONS","actual_fields":{"intent":"SEARCH_VIDEO_ACTIONS","countries":["ireland"],"country":"ireland","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"show me crashes","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_VIDEO_ACTIONS","actual_intent":"SEARCH_VIDEO_ACTIONS","actual_fields":{"intent":"SEARCH_VIDEO_ACTIONS","actionTypes":["crash"],"actionType":"crash","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"drifts of max freeman","category":"multi_filter","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_VIDEO_ACTIONS","actual_intent":"SEARCH_VIDEO_ACTIONS","actual_fields":{"intent":"SEARCH_VIDEO_ACTIONS","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","actionTypes":["drift"],"actionType":"drift","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"videos of max freeman","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_VIDEOS","actual_intent":"SEARCH_DRIVER_VIDEOS","actual_fields":{"intent":"SEARCH_DRIVER_VIDEOS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"watch max freeman onboard","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_VIDEOS","actual_intent":"SEARCH_DRIVER_VIDEOS","actual_fields":{"intent":"SEARCH_DRIVER_VIDEOS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"top uploaders","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_UPLOADERS","actual_intent":"GET_TOP_UPLOADERS","actual_fields":{"intent":"GET_TOP_UPLOADERS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"who uploaded the most videos","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_UPLOADERS","actual_intent":"GET_TOP_UPLOADERS","actual_fields":{"intent":"GET_TOP_UPLOADERS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"top drivers by wins","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_DRIVERS_BY_WINS","actual_intent":"GET_TOP_DRIVERS_BY_WINS","actual_fields":{"intent":"GET_TOP_DRIVERS_BY_WINS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"most successful drivers","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_DRIVERS_BY_WINS","actual_intent":"GET_TOP_DRIVERS_BY_WINS","actual_fields":{"intent":"GET_TOP_DRIVERS_BY_WINS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rally donegal","category":"ambiguity","expected_kind":"clarification","actual_kind":"clarification","expected_intent":null,"actual_intent":"SEARCH_RALLIES","actual_fields":null} +{"query":"who won rally donegal","category":"ambiguity","expected_kind":"clarification","actual_kind":"clarification","expected_intent":null,"actual_intent":"GET_RALLY_RESULTS","actual_fields":null} +{"query":"results for rally donegal","category":"ambiguity","expected_kind":"clarification","actual_kind":"clarification","expected_intent":null,"actual_intent":"GET_RALLY_TOP_FINISHERS","actual_fields":null} +{"query":"explain the offside rule in football","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what is the meaning of life","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"book me a flight to tokyo","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"rally zzzxqywv","category":"unsupported","expected_kind":"noMatch","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"compare aerodynamics of wrc cars in detail","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what is the weather","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"hello","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"thanks","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"who are you","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what can you do","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"tell me a joke","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"are you alive","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"who is the best rally driver of all time","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what is the capital of france","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-13-29-244605Z/offline_search_report.md b/backend/benchmarks/results/offline_search_2026-09-03T20-13-29-244605Z/offline_search_report.md new file mode 100644 index 0000000..af35b80 --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-13-29-244605Z/offline_search_report.md @@ -0,0 +1,29 @@ +# Offline Search Benchmark Report + +Generated: 2026-09-03T20-13-29-244605Z + +## Primary safety gate +- **wrong_confident: 0** (gate: must be 0) + +## Parser metrics +- Corpus size: 41 +- Intent accuracy: 100.0% (24/24) +- Field F1: 1.000 +- Entity resolution accuracy: 100.0% (14/14) +- Clarification accuracy: 100.0% (3/3) +- Safe unsupported rate: 100.0% (5/5) +- Special-query accuracy: 100.0% (9/9) +- OFFLINE_COVERAGE_RATE: 88.9% (24/27 answerable produced results) + +## Execution parity (offline SQLite vs online MySQL oracle) +- Cases matched: 16/16 + +## Connectivity fallback +- ONLINE: onlineAuthoritative +- OFFLINE: offlineLocal +- TIMEOUT: lowBandwidthLocal +- BACKEND_ERROR: backendUnreachableLocal + +## Voice offline +- Cloud voice offline: NO +- On-device voice offline: DEVICE_DEPENDENT diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-13-29-244605Z/snapshot_validation.json b/backend/benchmarks/results/offline_search_2026-09-03T20-13-29-244605Z/snapshot_validation.json new file mode 100644 index 0000000..0365224 --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-13-29-244605Z/snapshot_validation.json @@ -0,0 +1,14 @@ +{ + "snapshot_id": "1-4b19d6e0547dcab0-full", + "table_counts": { + "rallies": 111, + "people": 8750, + "stages": 1025, + "participation": 9967, + "final_results": 326, + "driver_wins": 5, + "uploader_stats": 259, + "video_meta": 18510, + "video_actions": 32497 + } +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-13-29-244605Z/special_query_results.json b/backend/benchmarks/results/offline_search_2026-09-03T20-13-29-244605Z/special_query_results.json new file mode 100644 index 0000000..279124a --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-13-29-244605Z/special_query_results.json @@ -0,0 +1,47 @@ +[ + { + "query": "what is the weather", + "category": "weather", + "intercepted": true + }, + { + "query": "hello", + "category": "greeting", + "intercepted": true + }, + { + "query": "thanks", + "category": "thanks", + "intercepted": true + }, + { + "query": "who are you", + "category": "identity", + "intercepted": true + }, + { + "query": "what can you do", + "category": "capabilities", + "intercepted": true + }, + { + "query": "tell me a joke", + "category": "joke", + "intercepted": true + }, + { + "query": "are you alive", + "category": "alive", + "intercepted": true + }, + { + "query": "who is the best rally driver of all time", + "category": "rallyOpinion", + "intercepted": true + }, + { + "query": "what is the capital of france", + "category": "unsupported", + "intercepted": true + } +] \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-13-29-244605Z/voice_offline_results.json b/backend/benchmarks/results/offline_search_2026-09-03T20-13-29-244605Z/voice_offline_results.json new file mode 100644 index 0000000..b99b8cd --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-13-29-244605Z/voice_offline_results.json @@ -0,0 +1,6 @@ +{ + "cloud_voice_offline": "NO (network required)", + "on_device_voice_offline": "DEVICE_DEPENDENT (only where OS on-device recognizer supports it)", + "auto_submit": false, + "transcript_editable": true +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-14-34-194066Z/connectivity_fallback_results.json b/backend/benchmarks/results/offline_search_2026-09-03T20-14-34-194066Z/connectivity_fallback_results.json new file mode 100644 index 0000000..e44473f --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-14-34-194066Z/connectivity_fallback_results.json @@ -0,0 +1,26 @@ +[ + { + "scenario": "ONLINE", + "mode": "onlineAuthoritative", + "ux_state": "online", + "silent_swap": false + }, + { + "scenario": "OFFLINE", + "mode": "offlineLocal", + "ux_state": "offlineLocalResults", + "silent_swap": false + }, + { + "scenario": "TIMEOUT", + "mode": "lowBandwidthLocal", + "ux_state": "lowBandwidthLocalFallback", + "silent_swap": false + }, + { + "scenario": "BACKEND_ERROR", + "mode": "backendUnreachableLocal", + "ux_state": "backendUnreachableLocalAvailable", + "silent_swap": false + } +] \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-14-34-194066Z/offline_benchmark_metadata.json b/backend/benchmarks/results/offline_search_2026-09-03T20-14-34-194066Z/offline_benchmark_metadata.json new file mode 100644 index 0000000..c45fa6e --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-14-34-194066Z/offline_benchmark_metadata.json @@ -0,0 +1,14 @@ +{ + "generated_at": "2026-09-03T20-14-34-194066Z", + "corpus_size": 41, + "intent_accuracy": 1.0, + "field_f1": 1.0, + "entity_accuracy": 1.0, + "clarification_accuracy": 1.0, + "safe_unsupported_rate": 1.0, + "special_accuracy": 1.0, + "wrong_confident": 0, + "offline_coverage_rate": 0.8888888888888888, + "execution_parity_pass": 16, + "execution_parity_total": 16 +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-14-34-194066Z/offline_execution_parity.jsonl b/backend/benchmarks/results/offline_search_2026-09-03T20-14-34-194066Z/offline_execution_parity.jsonl new file mode 100644 index 0000000..d5ce7ce --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-14-34-194066Z/offline_execution_parity.jsonl @@ -0,0 +1,16 @@ +{"name":"rallies_ireland_2025","intent":"SEARCH_RALLIES","online_total":24,"offline_total":24,"match":true} +{"name":"rallies_ireland_all","intent":"SEARCH_RALLIES","online_total":35,"offline_total":35,"match":true} +{"name":"rallies_portugal","intent":"SEARCH_RALLIES","online_total":7,"offline_total":7,"match":true} +{"name":"rallies_year_2026","intent":"SEARCH_RALLIES","online_total":54,"offline_total":54,"match":true} +{"name":"rally_aluksne_byid","intent":"SEARCH_RALLIES","online_total":1,"offline_total":1,"match":true} +{"name":"driver_rallies_freeman","intent":"SEARCH_DRIVER_RALLIES","online_total":9,"offline_total":9,"match":true} +{"name":"driver_wins_moffett","intent":"SEARCH_DRIVER_WINS","online_total":1,"offline_total":1,"match":true} +{"name":"rally_results_aluksne","intent":"GET_RALLY_RESULTS","online_total":1,"offline_total":1,"match":true} +{"name":"rally_results_donegal","intent":"GET_RALLY_RESULTS","online_total":1,"offline_total":1,"match":true} +{"name":"top_finishers_aluksne","intent":"GET_RALLY_TOP_FINISHERS","online_total":81,"offline_total":81,"match":true} +{"name":"top_finishers_donegal","intent":"GET_RALLY_TOP_FINISHERS","online_total":164,"offline_total":164,"match":true} +{"name":"top_uploaders_global","intent":"GET_TOP_UPLOADERS","online_total":259,"offline_total":259,"match":true} +{"name":"top_drivers_by_wins","intent":"GET_TOP_DRIVERS_BY_WINS","online_total":5,"offline_total":5,"match":true} +{"name":"driver_videos_freeman","intent":"SEARCH_DRIVER_VIDEOS","online_total":53,"offline_total":53,"match":true} +{"name":"video_actions_jumps","intent":"SEARCH_VIDEO_ACTIONS","online_total":3077,"offline_total":3077,"match":true} +{"name":"video_actions_crashes","intent":"SEARCH_VIDEO_ACTIONS","online_total":1627,"offline_total":1627,"match":true} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-14-34-194066Z/offline_failure_analysis.json b/backend/benchmarks/results/offline_search_2026-09-03T20-14-34-194066Z/offline_failure_analysis.json new file mode 100644 index 0000000..db8a271 --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-14-34-194066Z/offline_failure_analysis.json @@ -0,0 +1,4 @@ +{ + "wrong_confident": 0, + "failures": [] +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-14-34-194066Z/offline_parser_results.jsonl b/backend/benchmarks/results/offline_search_2026-09-03T20-14-34-194066Z/offline_parser_results.jsonl new file mode 100644 index 0000000..1308dba --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-14-34-194066Z/offline_parser_results.jsonl @@ -0,0 +1,41 @@ +{"query":"rallies in ireland in 2025","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","countries":["ireland"],"country":"ireland","years":[2025],"year":2025,"driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"what rallies happened in ireland","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","countries":["ireland"],"country":"ireland","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"portugal rallies 2024","category":"multi_filter","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","countries":["portugal"],"country":"portugal","years":[2024],"year":2024,"driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rally aluksne","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rally aluqsne","category":"typo","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rallies between 2024 and 2026","category":"multi_filter","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","yearFrom":2024,"yearTo":2026,"driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"max freeman rallies","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_RALLIES","actual_intent":"SEARCH_DRIVER_RALLIES","actual_fields":{"intent":"SEARCH_DRIVER_RALLIES","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"what rallies did max freeman compete in","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_RALLIES","actual_intent":"SEARCH_DRIVER_RALLIES","actual_fields":{"intent":"SEARCH_DRIVER_RALLIES","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"max freemn rallies","category":"typo","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_RALLIES","actual_intent":"SEARCH_DRIVER_RALLIES","actual_fields":{"intent":"SEARCH_DRIVER_RALLIES","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rallies won by max freeman","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_WINS","actual_intent":"SEARCH_DRIVER_WINS","actual_fields":{"intent":"SEARCH_DRIVER_WINS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"max freeman wins","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_WINS","actual_intent":"SEARCH_DRIVER_WINS","actual_fields":{"intent":"SEARCH_DRIVER_WINS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"who won rally aluksne","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_RESULTS","actual_intent":"GET_RALLY_RESULTS","actual_fields":{"intent":"GET_RALLY_RESULTS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"winner of rally aluksne","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_RESULTS","actual_intent":"GET_RALLY_RESULTS","actual_fields":{"intent":"GET_RALLY_RESULTS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"results for rally aluksne","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_TOP_FINISHERS","actual_intent":"GET_RALLY_TOP_FINISHERS","actual_fields":{"intent":"GET_RALLY_TOP_FINISHERS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"leaderboard for rally aluksne","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_TOP_FINISHERS","actual_intent":"GET_RALLY_TOP_FINISHERS","actual_fields":{"intent":"GET_RALLY_TOP_FINISHERS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"jumps from rally ireland","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_VIDEO_ACTIONS","actual_intent":"SEARCH_VIDEO_ACTIONS","actual_fields":{"intent":"SEARCH_VIDEO_ACTIONS","countries":["ireland"],"country":"ireland","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"show me crashes","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_VIDEO_ACTIONS","actual_intent":"SEARCH_VIDEO_ACTIONS","actual_fields":{"intent":"SEARCH_VIDEO_ACTIONS","actionTypes":["crash"],"actionType":"crash","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"drifts of max freeman","category":"multi_filter","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_VIDEO_ACTIONS","actual_intent":"SEARCH_VIDEO_ACTIONS","actual_fields":{"intent":"SEARCH_VIDEO_ACTIONS","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","actionTypes":["drift"],"actionType":"drift","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"videos of max freeman","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_VIDEOS","actual_intent":"SEARCH_DRIVER_VIDEOS","actual_fields":{"intent":"SEARCH_DRIVER_VIDEOS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"watch max freeman onboard","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_VIDEOS","actual_intent":"SEARCH_DRIVER_VIDEOS","actual_fields":{"intent":"SEARCH_DRIVER_VIDEOS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"top uploaders","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_UPLOADERS","actual_intent":"GET_TOP_UPLOADERS","actual_fields":{"intent":"GET_TOP_UPLOADERS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"who uploaded the most videos","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_UPLOADERS","actual_intent":"GET_TOP_UPLOADERS","actual_fields":{"intent":"GET_TOP_UPLOADERS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"top drivers by wins","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_DRIVERS_BY_WINS","actual_intent":"GET_TOP_DRIVERS_BY_WINS","actual_fields":{"intent":"GET_TOP_DRIVERS_BY_WINS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"most successful drivers","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_DRIVERS_BY_WINS","actual_intent":"GET_TOP_DRIVERS_BY_WINS","actual_fields":{"intent":"GET_TOP_DRIVERS_BY_WINS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rally donegal","category":"ambiguity","expected_kind":"clarification","actual_kind":"clarification","expected_intent":null,"actual_intent":"SEARCH_RALLIES","actual_fields":null} +{"query":"who won rally donegal","category":"ambiguity","expected_kind":"clarification","actual_kind":"clarification","expected_intent":null,"actual_intent":"GET_RALLY_RESULTS","actual_fields":null} +{"query":"results for rally donegal","category":"ambiguity","expected_kind":"clarification","actual_kind":"clarification","expected_intent":null,"actual_intent":"GET_RALLY_TOP_FINISHERS","actual_fields":null} +{"query":"explain the offside rule in football","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what is the meaning of life","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"book me a flight to tokyo","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"rally zzzxqywv","category":"unsupported","expected_kind":"noMatch","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"compare aerodynamics of wrc cars in detail","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what is the weather","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"hello","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"thanks","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"who are you","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what can you do","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"tell me a joke","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"are you alive","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"who is the best rally driver of all time","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what is the capital of france","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-14-34-194066Z/offline_search_report.md b/backend/benchmarks/results/offline_search_2026-09-03T20-14-34-194066Z/offline_search_report.md new file mode 100644 index 0000000..0386bb8 --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-14-34-194066Z/offline_search_report.md @@ -0,0 +1,29 @@ +# Offline Search Benchmark Report + +Generated: 2026-09-03T20-14-34-194066Z + +## Primary safety gate +- **wrong_confident: 0** (gate: must be 0) + +## Parser metrics +- Corpus size: 41 +- Intent accuracy: 100.0% (24/24) +- Field F1: 1.000 +- Entity resolution accuracy: 100.0% (14/14) +- Clarification accuracy: 100.0% (3/3) +- Safe unsupported rate: 100.0% (5/5) +- Special-query accuracy: 100.0% (9/9) +- OFFLINE_COVERAGE_RATE: 88.9% (24/27 answerable produced results) + +## Execution parity (offline SQLite vs online MySQL oracle) +- Cases matched: 16/16 + +## Connectivity fallback +- ONLINE: onlineAuthoritative +- OFFLINE: offlineLocal +- TIMEOUT: lowBandwidthLocal +- BACKEND_ERROR: backendUnreachableLocal + +## Voice offline +- Cloud voice offline: NO +- On-device voice offline: DEVICE_DEPENDENT diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-14-34-194066Z/snapshot_validation.json b/backend/benchmarks/results/offline_search_2026-09-03T20-14-34-194066Z/snapshot_validation.json new file mode 100644 index 0000000..0365224 --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-14-34-194066Z/snapshot_validation.json @@ -0,0 +1,14 @@ +{ + "snapshot_id": "1-4b19d6e0547dcab0-full", + "table_counts": { + "rallies": 111, + "people": 8750, + "stages": 1025, + "participation": 9967, + "final_results": 326, + "driver_wins": 5, + "uploader_stats": 259, + "video_meta": 18510, + "video_actions": 32497 + } +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-14-34-194066Z/special_query_results.json b/backend/benchmarks/results/offline_search_2026-09-03T20-14-34-194066Z/special_query_results.json new file mode 100644 index 0000000..279124a --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-14-34-194066Z/special_query_results.json @@ -0,0 +1,47 @@ +[ + { + "query": "what is the weather", + "category": "weather", + "intercepted": true + }, + { + "query": "hello", + "category": "greeting", + "intercepted": true + }, + { + "query": "thanks", + "category": "thanks", + "intercepted": true + }, + { + "query": "who are you", + "category": "identity", + "intercepted": true + }, + { + "query": "what can you do", + "category": "capabilities", + "intercepted": true + }, + { + "query": "tell me a joke", + "category": "joke", + "intercepted": true + }, + { + "query": "are you alive", + "category": "alive", + "intercepted": true + }, + { + "query": "who is the best rally driver of all time", + "category": "rallyOpinion", + "intercepted": true + }, + { + "query": "what is the capital of france", + "category": "unsupported", + "intercepted": true + } +] \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-14-34-194066Z/voice_offline_results.json b/backend/benchmarks/results/offline_search_2026-09-03T20-14-34-194066Z/voice_offline_results.json new file mode 100644 index 0000000..b99b8cd --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-14-34-194066Z/voice_offline_results.json @@ -0,0 +1,6 @@ +{ + "cloud_voice_offline": "NO (network required)", + "on_device_voice_offline": "DEVICE_DEPENDENT (only where OS on-device recognizer supports it)", + "auto_submit": false, + "transcript_editable": true +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-15-50-606577Z/connectivity_fallback_results.json b/backend/benchmarks/results/offline_search_2026-09-03T20-15-50-606577Z/connectivity_fallback_results.json new file mode 100644 index 0000000..e44473f --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-15-50-606577Z/connectivity_fallback_results.json @@ -0,0 +1,26 @@ +[ + { + "scenario": "ONLINE", + "mode": "onlineAuthoritative", + "ux_state": "online", + "silent_swap": false + }, + { + "scenario": "OFFLINE", + "mode": "offlineLocal", + "ux_state": "offlineLocalResults", + "silent_swap": false + }, + { + "scenario": "TIMEOUT", + "mode": "lowBandwidthLocal", + "ux_state": "lowBandwidthLocalFallback", + "silent_swap": false + }, + { + "scenario": "BACKEND_ERROR", + "mode": "backendUnreachableLocal", + "ux_state": "backendUnreachableLocalAvailable", + "silent_swap": false + } +] \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-15-50-606577Z/offline_benchmark_metadata.json b/backend/benchmarks/results/offline_search_2026-09-03T20-15-50-606577Z/offline_benchmark_metadata.json new file mode 100644 index 0000000..fe032be --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-15-50-606577Z/offline_benchmark_metadata.json @@ -0,0 +1,14 @@ +{ + "generated_at": "2026-09-03T20-15-50-606577Z", + "corpus_size": 41, + "intent_accuracy": 1.0, + "field_f1": 1.0, + "entity_accuracy": 1.0, + "clarification_accuracy": 1.0, + "safe_unsupported_rate": 1.0, + "special_accuracy": 1.0, + "wrong_confident": 0, + "offline_coverage_rate": 0.8888888888888888, + "execution_parity_pass": 16, + "execution_parity_total": 16 +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-15-50-606577Z/offline_execution_parity.jsonl b/backend/benchmarks/results/offline_search_2026-09-03T20-15-50-606577Z/offline_execution_parity.jsonl new file mode 100644 index 0000000..d5ce7ce --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-15-50-606577Z/offline_execution_parity.jsonl @@ -0,0 +1,16 @@ +{"name":"rallies_ireland_2025","intent":"SEARCH_RALLIES","online_total":24,"offline_total":24,"match":true} +{"name":"rallies_ireland_all","intent":"SEARCH_RALLIES","online_total":35,"offline_total":35,"match":true} +{"name":"rallies_portugal","intent":"SEARCH_RALLIES","online_total":7,"offline_total":7,"match":true} +{"name":"rallies_year_2026","intent":"SEARCH_RALLIES","online_total":54,"offline_total":54,"match":true} +{"name":"rally_aluksne_byid","intent":"SEARCH_RALLIES","online_total":1,"offline_total":1,"match":true} +{"name":"driver_rallies_freeman","intent":"SEARCH_DRIVER_RALLIES","online_total":9,"offline_total":9,"match":true} +{"name":"driver_wins_moffett","intent":"SEARCH_DRIVER_WINS","online_total":1,"offline_total":1,"match":true} +{"name":"rally_results_aluksne","intent":"GET_RALLY_RESULTS","online_total":1,"offline_total":1,"match":true} +{"name":"rally_results_donegal","intent":"GET_RALLY_RESULTS","online_total":1,"offline_total":1,"match":true} +{"name":"top_finishers_aluksne","intent":"GET_RALLY_TOP_FINISHERS","online_total":81,"offline_total":81,"match":true} +{"name":"top_finishers_donegal","intent":"GET_RALLY_TOP_FINISHERS","online_total":164,"offline_total":164,"match":true} +{"name":"top_uploaders_global","intent":"GET_TOP_UPLOADERS","online_total":259,"offline_total":259,"match":true} +{"name":"top_drivers_by_wins","intent":"GET_TOP_DRIVERS_BY_WINS","online_total":5,"offline_total":5,"match":true} +{"name":"driver_videos_freeman","intent":"SEARCH_DRIVER_VIDEOS","online_total":53,"offline_total":53,"match":true} +{"name":"video_actions_jumps","intent":"SEARCH_VIDEO_ACTIONS","online_total":3077,"offline_total":3077,"match":true} +{"name":"video_actions_crashes","intent":"SEARCH_VIDEO_ACTIONS","online_total":1627,"offline_total":1627,"match":true} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-15-50-606577Z/offline_failure_analysis.json b/backend/benchmarks/results/offline_search_2026-09-03T20-15-50-606577Z/offline_failure_analysis.json new file mode 100644 index 0000000..db8a271 --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-15-50-606577Z/offline_failure_analysis.json @@ -0,0 +1,4 @@ +{ + "wrong_confident": 0, + "failures": [] +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-15-50-606577Z/offline_parser_results.jsonl b/backend/benchmarks/results/offline_search_2026-09-03T20-15-50-606577Z/offline_parser_results.jsonl new file mode 100644 index 0000000..1308dba --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-15-50-606577Z/offline_parser_results.jsonl @@ -0,0 +1,41 @@ +{"query":"rallies in ireland in 2025","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","countries":["ireland"],"country":"ireland","years":[2025],"year":2025,"driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"what rallies happened in ireland","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","countries":["ireland"],"country":"ireland","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"portugal rallies 2024","category":"multi_filter","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","countries":["portugal"],"country":"portugal","years":[2024],"year":2024,"driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rally aluksne","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rally aluqsne","category":"typo","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rallies between 2024 and 2026","category":"multi_filter","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","yearFrom":2024,"yearTo":2026,"driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"max freeman rallies","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_RALLIES","actual_intent":"SEARCH_DRIVER_RALLIES","actual_fields":{"intent":"SEARCH_DRIVER_RALLIES","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"what rallies did max freeman compete in","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_RALLIES","actual_intent":"SEARCH_DRIVER_RALLIES","actual_fields":{"intent":"SEARCH_DRIVER_RALLIES","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"max freemn rallies","category":"typo","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_RALLIES","actual_intent":"SEARCH_DRIVER_RALLIES","actual_fields":{"intent":"SEARCH_DRIVER_RALLIES","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rallies won by max freeman","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_WINS","actual_intent":"SEARCH_DRIVER_WINS","actual_fields":{"intent":"SEARCH_DRIVER_WINS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"max freeman wins","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_WINS","actual_intent":"SEARCH_DRIVER_WINS","actual_fields":{"intent":"SEARCH_DRIVER_WINS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"who won rally aluksne","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_RESULTS","actual_intent":"GET_RALLY_RESULTS","actual_fields":{"intent":"GET_RALLY_RESULTS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"winner of rally aluksne","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_RESULTS","actual_intent":"GET_RALLY_RESULTS","actual_fields":{"intent":"GET_RALLY_RESULTS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"results for rally aluksne","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_TOP_FINISHERS","actual_intent":"GET_RALLY_TOP_FINISHERS","actual_fields":{"intent":"GET_RALLY_TOP_FINISHERS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"leaderboard for rally aluksne","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_TOP_FINISHERS","actual_intent":"GET_RALLY_TOP_FINISHERS","actual_fields":{"intent":"GET_RALLY_TOP_FINISHERS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"jumps from rally ireland","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_VIDEO_ACTIONS","actual_intent":"SEARCH_VIDEO_ACTIONS","actual_fields":{"intent":"SEARCH_VIDEO_ACTIONS","countries":["ireland"],"country":"ireland","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"show me crashes","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_VIDEO_ACTIONS","actual_intent":"SEARCH_VIDEO_ACTIONS","actual_fields":{"intent":"SEARCH_VIDEO_ACTIONS","actionTypes":["crash"],"actionType":"crash","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"drifts of max freeman","category":"multi_filter","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_VIDEO_ACTIONS","actual_intent":"SEARCH_VIDEO_ACTIONS","actual_fields":{"intent":"SEARCH_VIDEO_ACTIONS","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","actionTypes":["drift"],"actionType":"drift","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"videos of max freeman","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_VIDEOS","actual_intent":"SEARCH_DRIVER_VIDEOS","actual_fields":{"intent":"SEARCH_DRIVER_VIDEOS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"watch max freeman onboard","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_VIDEOS","actual_intent":"SEARCH_DRIVER_VIDEOS","actual_fields":{"intent":"SEARCH_DRIVER_VIDEOS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"top uploaders","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_UPLOADERS","actual_intent":"GET_TOP_UPLOADERS","actual_fields":{"intent":"GET_TOP_UPLOADERS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"who uploaded the most videos","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_UPLOADERS","actual_intent":"GET_TOP_UPLOADERS","actual_fields":{"intent":"GET_TOP_UPLOADERS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"top drivers by wins","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_DRIVERS_BY_WINS","actual_intent":"GET_TOP_DRIVERS_BY_WINS","actual_fields":{"intent":"GET_TOP_DRIVERS_BY_WINS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"most successful drivers","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_DRIVERS_BY_WINS","actual_intent":"GET_TOP_DRIVERS_BY_WINS","actual_fields":{"intent":"GET_TOP_DRIVERS_BY_WINS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rally donegal","category":"ambiguity","expected_kind":"clarification","actual_kind":"clarification","expected_intent":null,"actual_intent":"SEARCH_RALLIES","actual_fields":null} +{"query":"who won rally donegal","category":"ambiguity","expected_kind":"clarification","actual_kind":"clarification","expected_intent":null,"actual_intent":"GET_RALLY_RESULTS","actual_fields":null} +{"query":"results for rally donegal","category":"ambiguity","expected_kind":"clarification","actual_kind":"clarification","expected_intent":null,"actual_intent":"GET_RALLY_TOP_FINISHERS","actual_fields":null} +{"query":"explain the offside rule in football","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what is the meaning of life","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"book me a flight to tokyo","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"rally zzzxqywv","category":"unsupported","expected_kind":"noMatch","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"compare aerodynamics of wrc cars in detail","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what is the weather","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"hello","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"thanks","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"who are you","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what can you do","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"tell me a joke","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"are you alive","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"who is the best rally driver of all time","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what is the capital of france","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-15-50-606577Z/offline_search_report.md b/backend/benchmarks/results/offline_search_2026-09-03T20-15-50-606577Z/offline_search_report.md new file mode 100644 index 0000000..9e32efc --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-15-50-606577Z/offline_search_report.md @@ -0,0 +1,29 @@ +# Offline Search Benchmark Report + +Generated: 2026-09-03T20-15-50-606577Z + +## Primary safety gate +- **wrong_confident: 0** (gate: must be 0) + +## Parser metrics +- Corpus size: 41 +- Intent accuracy: 100.0% (24/24) +- Field F1: 1.000 +- Entity resolution accuracy: 100.0% (14/14) +- Clarification accuracy: 100.0% (3/3) +- Safe unsupported rate: 100.0% (5/5) +- Special-query accuracy: 100.0% (9/9) +- OFFLINE_COVERAGE_RATE: 88.9% (24/27 answerable produced results) + +## Execution parity (offline SQLite vs online MySQL oracle) +- Cases matched: 16/16 + +## Connectivity fallback +- ONLINE: onlineAuthoritative +- OFFLINE: offlineLocal +- TIMEOUT: lowBandwidthLocal +- BACKEND_ERROR: backendUnreachableLocal + +## Voice offline +- Cloud voice offline: NO +- On-device voice offline: DEVICE_DEPENDENT diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-15-50-606577Z/snapshot_validation.json b/backend/benchmarks/results/offline_search_2026-09-03T20-15-50-606577Z/snapshot_validation.json new file mode 100644 index 0000000..0365224 --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-15-50-606577Z/snapshot_validation.json @@ -0,0 +1,14 @@ +{ + "snapshot_id": "1-4b19d6e0547dcab0-full", + "table_counts": { + "rallies": 111, + "people": 8750, + "stages": 1025, + "participation": 9967, + "final_results": 326, + "driver_wins": 5, + "uploader_stats": 259, + "video_meta": 18510, + "video_actions": 32497 + } +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-15-50-606577Z/special_query_results.json b/backend/benchmarks/results/offline_search_2026-09-03T20-15-50-606577Z/special_query_results.json new file mode 100644 index 0000000..279124a --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-15-50-606577Z/special_query_results.json @@ -0,0 +1,47 @@ +[ + { + "query": "what is the weather", + "category": "weather", + "intercepted": true + }, + { + "query": "hello", + "category": "greeting", + "intercepted": true + }, + { + "query": "thanks", + "category": "thanks", + "intercepted": true + }, + { + "query": "who are you", + "category": "identity", + "intercepted": true + }, + { + "query": "what can you do", + "category": "capabilities", + "intercepted": true + }, + { + "query": "tell me a joke", + "category": "joke", + "intercepted": true + }, + { + "query": "are you alive", + "category": "alive", + "intercepted": true + }, + { + "query": "who is the best rally driver of all time", + "category": "rallyOpinion", + "intercepted": true + }, + { + "query": "what is the capital of france", + "category": "unsupported", + "intercepted": true + } +] \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-15-50-606577Z/voice_offline_results.json b/backend/benchmarks/results/offline_search_2026-09-03T20-15-50-606577Z/voice_offline_results.json new file mode 100644 index 0000000..b99b8cd --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-15-50-606577Z/voice_offline_results.json @@ -0,0 +1,6 @@ +{ + "cloud_voice_offline": "NO (network required)", + "on_device_voice_offline": "DEVICE_DEPENDENT (only where OS on-device recognizer supports it)", + "auto_submit": false, + "transcript_editable": true +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-20-15-257129Z/connectivity_fallback_results.json b/backend/benchmarks/results/offline_search_2026-09-03T20-20-15-257129Z/connectivity_fallback_results.json new file mode 100644 index 0000000..e44473f --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-20-15-257129Z/connectivity_fallback_results.json @@ -0,0 +1,26 @@ +[ + { + "scenario": "ONLINE", + "mode": "onlineAuthoritative", + "ux_state": "online", + "silent_swap": false + }, + { + "scenario": "OFFLINE", + "mode": "offlineLocal", + "ux_state": "offlineLocalResults", + "silent_swap": false + }, + { + "scenario": "TIMEOUT", + "mode": "lowBandwidthLocal", + "ux_state": "lowBandwidthLocalFallback", + "silent_swap": false + }, + { + "scenario": "BACKEND_ERROR", + "mode": "backendUnreachableLocal", + "ux_state": "backendUnreachableLocalAvailable", + "silent_swap": false + } +] \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-20-15-257129Z/offline_benchmark_metadata.json b/backend/benchmarks/results/offline_search_2026-09-03T20-20-15-257129Z/offline_benchmark_metadata.json new file mode 100644 index 0000000..656d12d --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-20-15-257129Z/offline_benchmark_metadata.json @@ -0,0 +1,14 @@ +{ + "generated_at": "2026-09-03T20-20-15-257129Z", + "corpus_size": 41, + "intent_accuracy": 1.0, + "field_f1": 1.0, + "entity_accuracy": 1.0, + "clarification_accuracy": 1.0, + "safe_unsupported_rate": 1.0, + "special_accuracy": 1.0, + "wrong_confident": 0, + "offline_coverage_rate": 0.8888888888888888, + "execution_parity_pass": 16, + "execution_parity_total": 16 +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-20-15-257129Z/offline_execution_parity.jsonl b/backend/benchmarks/results/offline_search_2026-09-03T20-20-15-257129Z/offline_execution_parity.jsonl new file mode 100644 index 0000000..d5ce7ce --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-20-15-257129Z/offline_execution_parity.jsonl @@ -0,0 +1,16 @@ +{"name":"rallies_ireland_2025","intent":"SEARCH_RALLIES","online_total":24,"offline_total":24,"match":true} +{"name":"rallies_ireland_all","intent":"SEARCH_RALLIES","online_total":35,"offline_total":35,"match":true} +{"name":"rallies_portugal","intent":"SEARCH_RALLIES","online_total":7,"offline_total":7,"match":true} +{"name":"rallies_year_2026","intent":"SEARCH_RALLIES","online_total":54,"offline_total":54,"match":true} +{"name":"rally_aluksne_byid","intent":"SEARCH_RALLIES","online_total":1,"offline_total":1,"match":true} +{"name":"driver_rallies_freeman","intent":"SEARCH_DRIVER_RALLIES","online_total":9,"offline_total":9,"match":true} +{"name":"driver_wins_moffett","intent":"SEARCH_DRIVER_WINS","online_total":1,"offline_total":1,"match":true} +{"name":"rally_results_aluksne","intent":"GET_RALLY_RESULTS","online_total":1,"offline_total":1,"match":true} +{"name":"rally_results_donegal","intent":"GET_RALLY_RESULTS","online_total":1,"offline_total":1,"match":true} +{"name":"top_finishers_aluksne","intent":"GET_RALLY_TOP_FINISHERS","online_total":81,"offline_total":81,"match":true} +{"name":"top_finishers_donegal","intent":"GET_RALLY_TOP_FINISHERS","online_total":164,"offline_total":164,"match":true} +{"name":"top_uploaders_global","intent":"GET_TOP_UPLOADERS","online_total":259,"offline_total":259,"match":true} +{"name":"top_drivers_by_wins","intent":"GET_TOP_DRIVERS_BY_WINS","online_total":5,"offline_total":5,"match":true} +{"name":"driver_videos_freeman","intent":"SEARCH_DRIVER_VIDEOS","online_total":53,"offline_total":53,"match":true} +{"name":"video_actions_jumps","intent":"SEARCH_VIDEO_ACTIONS","online_total":3077,"offline_total":3077,"match":true} +{"name":"video_actions_crashes","intent":"SEARCH_VIDEO_ACTIONS","online_total":1627,"offline_total":1627,"match":true} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-20-15-257129Z/offline_failure_analysis.json b/backend/benchmarks/results/offline_search_2026-09-03T20-20-15-257129Z/offline_failure_analysis.json new file mode 100644 index 0000000..db8a271 --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-20-15-257129Z/offline_failure_analysis.json @@ -0,0 +1,4 @@ +{ + "wrong_confident": 0, + "failures": [] +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-20-15-257129Z/offline_parser_results.jsonl b/backend/benchmarks/results/offline_search_2026-09-03T20-20-15-257129Z/offline_parser_results.jsonl new file mode 100644 index 0000000..1308dba --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-20-15-257129Z/offline_parser_results.jsonl @@ -0,0 +1,41 @@ +{"query":"rallies in ireland in 2025","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","countries":["ireland"],"country":"ireland","years":[2025],"year":2025,"driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"what rallies happened in ireland","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","countries":["ireland"],"country":"ireland","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"portugal rallies 2024","category":"multi_filter","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","countries":["portugal"],"country":"portugal","years":[2024],"year":2024,"driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rally aluksne","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rally aluqsne","category":"typo","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rallies between 2024 and 2026","category":"multi_filter","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","yearFrom":2024,"yearTo":2026,"driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"max freeman rallies","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_RALLIES","actual_intent":"SEARCH_DRIVER_RALLIES","actual_fields":{"intent":"SEARCH_DRIVER_RALLIES","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"what rallies did max freeman compete in","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_RALLIES","actual_intent":"SEARCH_DRIVER_RALLIES","actual_fields":{"intent":"SEARCH_DRIVER_RALLIES","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"max freemn rallies","category":"typo","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_RALLIES","actual_intent":"SEARCH_DRIVER_RALLIES","actual_fields":{"intent":"SEARCH_DRIVER_RALLIES","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rallies won by max freeman","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_WINS","actual_intent":"SEARCH_DRIVER_WINS","actual_fields":{"intent":"SEARCH_DRIVER_WINS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"max freeman wins","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_WINS","actual_intent":"SEARCH_DRIVER_WINS","actual_fields":{"intent":"SEARCH_DRIVER_WINS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"who won rally aluksne","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_RESULTS","actual_intent":"GET_RALLY_RESULTS","actual_fields":{"intent":"GET_RALLY_RESULTS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"winner of rally aluksne","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_RESULTS","actual_intent":"GET_RALLY_RESULTS","actual_fields":{"intent":"GET_RALLY_RESULTS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"results for rally aluksne","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_TOP_FINISHERS","actual_intent":"GET_RALLY_TOP_FINISHERS","actual_fields":{"intent":"GET_RALLY_TOP_FINISHERS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"leaderboard for rally aluksne","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_TOP_FINISHERS","actual_intent":"GET_RALLY_TOP_FINISHERS","actual_fields":{"intent":"GET_RALLY_TOP_FINISHERS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"jumps from rally ireland","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_VIDEO_ACTIONS","actual_intent":"SEARCH_VIDEO_ACTIONS","actual_fields":{"intent":"SEARCH_VIDEO_ACTIONS","countries":["ireland"],"country":"ireland","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"show me crashes","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_VIDEO_ACTIONS","actual_intent":"SEARCH_VIDEO_ACTIONS","actual_fields":{"intent":"SEARCH_VIDEO_ACTIONS","actionTypes":["crash"],"actionType":"crash","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"drifts of max freeman","category":"multi_filter","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_VIDEO_ACTIONS","actual_intent":"SEARCH_VIDEO_ACTIONS","actual_fields":{"intent":"SEARCH_VIDEO_ACTIONS","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","actionTypes":["drift"],"actionType":"drift","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"videos of max freeman","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_VIDEOS","actual_intent":"SEARCH_DRIVER_VIDEOS","actual_fields":{"intent":"SEARCH_DRIVER_VIDEOS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"watch max freeman onboard","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_VIDEOS","actual_intent":"SEARCH_DRIVER_VIDEOS","actual_fields":{"intent":"SEARCH_DRIVER_VIDEOS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"top uploaders","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_UPLOADERS","actual_intent":"GET_TOP_UPLOADERS","actual_fields":{"intent":"GET_TOP_UPLOADERS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"who uploaded the most videos","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_UPLOADERS","actual_intent":"GET_TOP_UPLOADERS","actual_fields":{"intent":"GET_TOP_UPLOADERS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"top drivers by wins","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_DRIVERS_BY_WINS","actual_intent":"GET_TOP_DRIVERS_BY_WINS","actual_fields":{"intent":"GET_TOP_DRIVERS_BY_WINS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"most successful drivers","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_DRIVERS_BY_WINS","actual_intent":"GET_TOP_DRIVERS_BY_WINS","actual_fields":{"intent":"GET_TOP_DRIVERS_BY_WINS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rally donegal","category":"ambiguity","expected_kind":"clarification","actual_kind":"clarification","expected_intent":null,"actual_intent":"SEARCH_RALLIES","actual_fields":null} +{"query":"who won rally donegal","category":"ambiguity","expected_kind":"clarification","actual_kind":"clarification","expected_intent":null,"actual_intent":"GET_RALLY_RESULTS","actual_fields":null} +{"query":"results for rally donegal","category":"ambiguity","expected_kind":"clarification","actual_kind":"clarification","expected_intent":null,"actual_intent":"GET_RALLY_TOP_FINISHERS","actual_fields":null} +{"query":"explain the offside rule in football","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what is the meaning of life","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"book me a flight to tokyo","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"rally zzzxqywv","category":"unsupported","expected_kind":"noMatch","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"compare aerodynamics of wrc cars in detail","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what is the weather","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"hello","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"thanks","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"who are you","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what can you do","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"tell me a joke","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"are you alive","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"who is the best rally driver of all time","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what is the capital of france","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-20-15-257129Z/offline_search_report.md b/backend/benchmarks/results/offline_search_2026-09-03T20-20-15-257129Z/offline_search_report.md new file mode 100644 index 0000000..7ae4eb7 --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-20-15-257129Z/offline_search_report.md @@ -0,0 +1,29 @@ +# Offline Search Benchmark Report + +Generated: 2026-09-03T20-20-15-257129Z + +## Primary safety gate +- **wrong_confident: 0** (gate: must be 0) + +## Parser metrics +- Corpus size: 41 +- Intent accuracy: 100.0% (24/24) +- Field F1: 1.000 +- Entity resolution accuracy: 100.0% (14/14) +- Clarification accuracy: 100.0% (3/3) +- Safe unsupported rate: 100.0% (5/5) +- Special-query accuracy: 100.0% (9/9) +- OFFLINE_COVERAGE_RATE: 88.9% (24/27 answerable produced results) + +## Execution parity (offline SQLite vs online MySQL oracle) +- Cases matched: 16/16 + +## Connectivity fallback +- ONLINE: onlineAuthoritative +- OFFLINE: offlineLocal +- TIMEOUT: lowBandwidthLocal +- BACKEND_ERROR: backendUnreachableLocal + +## Voice offline +- Cloud voice offline: NO +- On-device voice offline: DEVICE_DEPENDENT diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-20-15-257129Z/snapshot_validation.json b/backend/benchmarks/results/offline_search_2026-09-03T20-20-15-257129Z/snapshot_validation.json new file mode 100644 index 0000000..0365224 --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-20-15-257129Z/snapshot_validation.json @@ -0,0 +1,14 @@ +{ + "snapshot_id": "1-4b19d6e0547dcab0-full", + "table_counts": { + "rallies": 111, + "people": 8750, + "stages": 1025, + "participation": 9967, + "final_results": 326, + "driver_wins": 5, + "uploader_stats": 259, + "video_meta": 18510, + "video_actions": 32497 + } +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-20-15-257129Z/special_query_results.json b/backend/benchmarks/results/offline_search_2026-09-03T20-20-15-257129Z/special_query_results.json new file mode 100644 index 0000000..279124a --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-20-15-257129Z/special_query_results.json @@ -0,0 +1,47 @@ +[ + { + "query": "what is the weather", + "category": "weather", + "intercepted": true + }, + { + "query": "hello", + "category": "greeting", + "intercepted": true + }, + { + "query": "thanks", + "category": "thanks", + "intercepted": true + }, + { + "query": "who are you", + "category": "identity", + "intercepted": true + }, + { + "query": "what can you do", + "category": "capabilities", + "intercepted": true + }, + { + "query": "tell me a joke", + "category": "joke", + "intercepted": true + }, + { + "query": "are you alive", + "category": "alive", + "intercepted": true + }, + { + "query": "who is the best rally driver of all time", + "category": "rallyOpinion", + "intercepted": true + }, + { + "query": "what is the capital of france", + "category": "unsupported", + "intercepted": true + } +] \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T20-20-15-257129Z/voice_offline_results.json b/backend/benchmarks/results/offline_search_2026-09-03T20-20-15-257129Z/voice_offline_results.json new file mode 100644 index 0000000..b99b8cd --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T20-20-15-257129Z/voice_offline_results.json @@ -0,0 +1,6 @@ +{ + "cloud_voice_offline": "NO (network required)", + "on_device_voice_offline": "DEVICE_DEPENDENT (only where OS on-device recognizer supports it)", + "auto_submit": false, + "transcript_editable": true +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T23-34-20-920659Z/connectivity_fallback_results.json b/backend/benchmarks/results/offline_search_2026-09-03T23-34-20-920659Z/connectivity_fallback_results.json new file mode 100644 index 0000000..a970336 --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T23-34-20-920659Z/connectivity_fallback_results.json @@ -0,0 +1,35 @@ +[ + { + "scenario": "ONLINE", + "stages": [ + "online" + ], + "result_source": "online", + "silent_swap": false + }, + { + "scenario": "OFFLINE", + "stages": [ + "offlineImmediate" + ], + "result_source": "offline", + "silent_swap": false + }, + { + "scenario": "TIMEOUT", + "stages": [ + "offlineFallback", + "lateOnlineAvailable" + ], + "result_source": "offline_fallback", + "silent_swap": false + }, + { + "scenario": "BACKEND_ERROR", + "stages": [ + "offlineAfterOnlineFailure" + ], + "result_source": "offline_fallback", + "silent_swap": false + } +] \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T23-34-20-920659Z/offline_benchmark_metadata.json b/backend/benchmarks/results/offline_search_2026-09-03T23-34-20-920659Z/offline_benchmark_metadata.json new file mode 100644 index 0000000..37ec848 --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T23-34-20-920659Z/offline_benchmark_metadata.json @@ -0,0 +1,14 @@ +{ + "generated_at": "2026-09-03T23-34-20-920659Z", + "corpus_size": 41, + "intent_accuracy": 1.0, + "field_f1": 1.0, + "entity_accuracy": 1.0, + "clarification_accuracy": 1.0, + "safe_unsupported_rate": 1.0, + "special_accuracy": 1.0, + "wrong_confident": 0, + "offline_coverage_rate": 0.8888888888888888, + "execution_parity_pass": 16, + "execution_parity_total": 16 +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T23-34-20-920659Z/offline_execution_parity.jsonl b/backend/benchmarks/results/offline_search_2026-09-03T23-34-20-920659Z/offline_execution_parity.jsonl new file mode 100644 index 0000000..d5ce7ce --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T23-34-20-920659Z/offline_execution_parity.jsonl @@ -0,0 +1,16 @@ +{"name":"rallies_ireland_2025","intent":"SEARCH_RALLIES","online_total":24,"offline_total":24,"match":true} +{"name":"rallies_ireland_all","intent":"SEARCH_RALLIES","online_total":35,"offline_total":35,"match":true} +{"name":"rallies_portugal","intent":"SEARCH_RALLIES","online_total":7,"offline_total":7,"match":true} +{"name":"rallies_year_2026","intent":"SEARCH_RALLIES","online_total":54,"offline_total":54,"match":true} +{"name":"rally_aluksne_byid","intent":"SEARCH_RALLIES","online_total":1,"offline_total":1,"match":true} +{"name":"driver_rallies_freeman","intent":"SEARCH_DRIVER_RALLIES","online_total":9,"offline_total":9,"match":true} +{"name":"driver_wins_moffett","intent":"SEARCH_DRIVER_WINS","online_total":1,"offline_total":1,"match":true} +{"name":"rally_results_aluksne","intent":"GET_RALLY_RESULTS","online_total":1,"offline_total":1,"match":true} +{"name":"rally_results_donegal","intent":"GET_RALLY_RESULTS","online_total":1,"offline_total":1,"match":true} +{"name":"top_finishers_aluksne","intent":"GET_RALLY_TOP_FINISHERS","online_total":81,"offline_total":81,"match":true} +{"name":"top_finishers_donegal","intent":"GET_RALLY_TOP_FINISHERS","online_total":164,"offline_total":164,"match":true} +{"name":"top_uploaders_global","intent":"GET_TOP_UPLOADERS","online_total":259,"offline_total":259,"match":true} +{"name":"top_drivers_by_wins","intent":"GET_TOP_DRIVERS_BY_WINS","online_total":5,"offline_total":5,"match":true} +{"name":"driver_videos_freeman","intent":"SEARCH_DRIVER_VIDEOS","online_total":53,"offline_total":53,"match":true} +{"name":"video_actions_jumps","intent":"SEARCH_VIDEO_ACTIONS","online_total":3077,"offline_total":3077,"match":true} +{"name":"video_actions_crashes","intent":"SEARCH_VIDEO_ACTIONS","online_total":1627,"offline_total":1627,"match":true} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T23-34-20-920659Z/offline_failure_analysis.json b/backend/benchmarks/results/offline_search_2026-09-03T23-34-20-920659Z/offline_failure_analysis.json new file mode 100644 index 0000000..db8a271 --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T23-34-20-920659Z/offline_failure_analysis.json @@ -0,0 +1,4 @@ +{ + "wrong_confident": 0, + "failures": [] +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T23-34-20-920659Z/offline_parser_results.jsonl b/backend/benchmarks/results/offline_search_2026-09-03T23-34-20-920659Z/offline_parser_results.jsonl new file mode 100644 index 0000000..1308dba --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T23-34-20-920659Z/offline_parser_results.jsonl @@ -0,0 +1,41 @@ +{"query":"rallies in ireland in 2025","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","countries":["ireland"],"country":"ireland","years":[2025],"year":2025,"driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"what rallies happened in ireland","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","countries":["ireland"],"country":"ireland","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"portugal rallies 2024","category":"multi_filter","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","countries":["portugal"],"country":"portugal","years":[2024],"year":2024,"driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rally aluksne","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rally aluqsne","category":"typo","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rallies between 2024 and 2026","category":"multi_filter","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","yearFrom":2024,"yearTo":2026,"driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"max freeman rallies","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_RALLIES","actual_intent":"SEARCH_DRIVER_RALLIES","actual_fields":{"intent":"SEARCH_DRIVER_RALLIES","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"what rallies did max freeman compete in","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_RALLIES","actual_intent":"SEARCH_DRIVER_RALLIES","actual_fields":{"intent":"SEARCH_DRIVER_RALLIES","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"max freemn rallies","category":"typo","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_RALLIES","actual_intent":"SEARCH_DRIVER_RALLIES","actual_fields":{"intent":"SEARCH_DRIVER_RALLIES","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rallies won by max freeman","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_WINS","actual_intent":"SEARCH_DRIVER_WINS","actual_fields":{"intent":"SEARCH_DRIVER_WINS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"max freeman wins","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_WINS","actual_intent":"SEARCH_DRIVER_WINS","actual_fields":{"intent":"SEARCH_DRIVER_WINS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"who won rally aluksne","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_RESULTS","actual_intent":"GET_RALLY_RESULTS","actual_fields":{"intent":"GET_RALLY_RESULTS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"winner of rally aluksne","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_RESULTS","actual_intent":"GET_RALLY_RESULTS","actual_fields":{"intent":"GET_RALLY_RESULTS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"results for rally aluksne","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_TOP_FINISHERS","actual_intent":"GET_RALLY_TOP_FINISHERS","actual_fields":{"intent":"GET_RALLY_TOP_FINISHERS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"leaderboard for rally aluksne","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_TOP_FINISHERS","actual_intent":"GET_RALLY_TOP_FINISHERS","actual_fields":{"intent":"GET_RALLY_TOP_FINISHERS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"jumps from rally ireland","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_VIDEO_ACTIONS","actual_intent":"SEARCH_VIDEO_ACTIONS","actual_fields":{"intent":"SEARCH_VIDEO_ACTIONS","countries":["ireland"],"country":"ireland","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"show me crashes","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_VIDEO_ACTIONS","actual_intent":"SEARCH_VIDEO_ACTIONS","actual_fields":{"intent":"SEARCH_VIDEO_ACTIONS","actionTypes":["crash"],"actionType":"crash","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"drifts of max freeman","category":"multi_filter","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_VIDEO_ACTIONS","actual_intent":"SEARCH_VIDEO_ACTIONS","actual_fields":{"intent":"SEARCH_VIDEO_ACTIONS","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","actionTypes":["drift"],"actionType":"drift","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"videos of max freeman","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_VIDEOS","actual_intent":"SEARCH_DRIVER_VIDEOS","actual_fields":{"intent":"SEARCH_DRIVER_VIDEOS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"watch max freeman onboard","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_VIDEOS","actual_intent":"SEARCH_DRIVER_VIDEOS","actual_fields":{"intent":"SEARCH_DRIVER_VIDEOS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"top uploaders","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_UPLOADERS","actual_intent":"GET_TOP_UPLOADERS","actual_fields":{"intent":"GET_TOP_UPLOADERS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"who uploaded the most videos","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_UPLOADERS","actual_intent":"GET_TOP_UPLOADERS","actual_fields":{"intent":"GET_TOP_UPLOADERS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"top drivers by wins","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_DRIVERS_BY_WINS","actual_intent":"GET_TOP_DRIVERS_BY_WINS","actual_fields":{"intent":"GET_TOP_DRIVERS_BY_WINS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"most successful drivers","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_DRIVERS_BY_WINS","actual_intent":"GET_TOP_DRIVERS_BY_WINS","actual_fields":{"intent":"GET_TOP_DRIVERS_BY_WINS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rally donegal","category":"ambiguity","expected_kind":"clarification","actual_kind":"clarification","expected_intent":null,"actual_intent":"SEARCH_RALLIES","actual_fields":null} +{"query":"who won rally donegal","category":"ambiguity","expected_kind":"clarification","actual_kind":"clarification","expected_intent":null,"actual_intent":"GET_RALLY_RESULTS","actual_fields":null} +{"query":"results for rally donegal","category":"ambiguity","expected_kind":"clarification","actual_kind":"clarification","expected_intent":null,"actual_intent":"GET_RALLY_TOP_FINISHERS","actual_fields":null} +{"query":"explain the offside rule in football","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what is the meaning of life","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"book me a flight to tokyo","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"rally zzzxqywv","category":"unsupported","expected_kind":"noMatch","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"compare aerodynamics of wrc cars in detail","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what is the weather","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"hello","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"thanks","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"who are you","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what can you do","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"tell me a joke","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"are you alive","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"who is the best rally driver of all time","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what is the capital of france","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T23-34-20-920659Z/offline_search_report.md b/backend/benchmarks/results/offline_search_2026-09-03T23-34-20-920659Z/offline_search_report.md new file mode 100644 index 0000000..8cc6f0d --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T23-34-20-920659Z/offline_search_report.md @@ -0,0 +1,29 @@ +# Offline Search Benchmark Report + +Generated: 2026-09-03T23-34-20-920659Z + +## Primary safety gate +- **wrong_confident: 0** (gate: must be 0) + +## Parser metrics +- Corpus size: 41 +- Intent accuracy: 100.0% (24/24) +- Field F1: 1.000 +- Entity resolution accuracy: 100.0% (14/14) +- Clarification accuracy: 100.0% (3/3) +- Safe unsupported rate: 100.0% (5/5) +- Special-query accuracy: 100.0% (9/9) +- OFFLINE_COVERAGE_RATE: 88.9% (24/27 answerable produced results) + +## Execution parity (offline SQLite vs online MySQL oracle) +- Cases matched: 16/16 + +## Connectivity fallback +- ONLINE: null +- OFFLINE: null +- TIMEOUT: null +- BACKEND_ERROR: null + +## Voice offline +- Cloud voice offline: NO +- On-device voice offline: DEVICE_DEPENDENT diff --git a/backend/benchmarks/results/offline_search_2026-09-03T23-34-20-920659Z/snapshot_validation.json b/backend/benchmarks/results/offline_search_2026-09-03T23-34-20-920659Z/snapshot_validation.json new file mode 100644 index 0000000..0365224 --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T23-34-20-920659Z/snapshot_validation.json @@ -0,0 +1,14 @@ +{ + "snapshot_id": "1-4b19d6e0547dcab0-full", + "table_counts": { + "rallies": 111, + "people": 8750, + "stages": 1025, + "participation": 9967, + "final_results": 326, + "driver_wins": 5, + "uploader_stats": 259, + "video_meta": 18510, + "video_actions": 32497 + } +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T23-34-20-920659Z/special_query_results.json b/backend/benchmarks/results/offline_search_2026-09-03T23-34-20-920659Z/special_query_results.json new file mode 100644 index 0000000..279124a --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T23-34-20-920659Z/special_query_results.json @@ -0,0 +1,47 @@ +[ + { + "query": "what is the weather", + "category": "weather", + "intercepted": true + }, + { + "query": "hello", + "category": "greeting", + "intercepted": true + }, + { + "query": "thanks", + "category": "thanks", + "intercepted": true + }, + { + "query": "who are you", + "category": "identity", + "intercepted": true + }, + { + "query": "what can you do", + "category": "capabilities", + "intercepted": true + }, + { + "query": "tell me a joke", + "category": "joke", + "intercepted": true + }, + { + "query": "are you alive", + "category": "alive", + "intercepted": true + }, + { + "query": "who is the best rally driver of all time", + "category": "rallyOpinion", + "intercepted": true + }, + { + "query": "what is the capital of france", + "category": "unsupported", + "intercepted": true + } +] \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T23-34-20-920659Z/voice_offline_results.json b/backend/benchmarks/results/offline_search_2026-09-03T23-34-20-920659Z/voice_offline_results.json new file mode 100644 index 0000000..b99b8cd --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T23-34-20-920659Z/voice_offline_results.json @@ -0,0 +1,6 @@ +{ + "cloud_voice_offline": "NO (network required)", + "on_device_voice_offline": "DEVICE_DEPENDENT (only where OS on-device recognizer supports it)", + "auto_submit": false, + "transcript_editable": true +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T23-35-52-240892Z/connectivity_fallback_results.json b/backend/benchmarks/results/offline_search_2026-09-03T23-35-52-240892Z/connectivity_fallback_results.json new file mode 100644 index 0000000..a970336 --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T23-35-52-240892Z/connectivity_fallback_results.json @@ -0,0 +1,35 @@ +[ + { + "scenario": "ONLINE", + "stages": [ + "online" + ], + "result_source": "online", + "silent_swap": false + }, + { + "scenario": "OFFLINE", + "stages": [ + "offlineImmediate" + ], + "result_source": "offline", + "silent_swap": false + }, + { + "scenario": "TIMEOUT", + "stages": [ + "offlineFallback", + "lateOnlineAvailable" + ], + "result_source": "offline_fallback", + "silent_swap": false + }, + { + "scenario": "BACKEND_ERROR", + "stages": [ + "offlineAfterOnlineFailure" + ], + "result_source": "offline_fallback", + "silent_swap": false + } +] \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T23-35-52-240892Z/offline_benchmark_metadata.json b/backend/benchmarks/results/offline_search_2026-09-03T23-35-52-240892Z/offline_benchmark_metadata.json new file mode 100644 index 0000000..1c05978 --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T23-35-52-240892Z/offline_benchmark_metadata.json @@ -0,0 +1,14 @@ +{ + "generated_at": "2026-09-03T23-35-52-240892Z", + "corpus_size": 41, + "intent_accuracy": 1.0, + "field_f1": 1.0, + "entity_accuracy": 1.0, + "clarification_accuracy": 1.0, + "safe_unsupported_rate": 1.0, + "special_accuracy": 1.0, + "wrong_confident": 0, + "offline_coverage_rate": 0.8888888888888888, + "execution_parity_pass": 16, + "execution_parity_total": 16 +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T23-35-52-240892Z/offline_execution_parity.jsonl b/backend/benchmarks/results/offline_search_2026-09-03T23-35-52-240892Z/offline_execution_parity.jsonl new file mode 100644 index 0000000..d5ce7ce --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T23-35-52-240892Z/offline_execution_parity.jsonl @@ -0,0 +1,16 @@ +{"name":"rallies_ireland_2025","intent":"SEARCH_RALLIES","online_total":24,"offline_total":24,"match":true} +{"name":"rallies_ireland_all","intent":"SEARCH_RALLIES","online_total":35,"offline_total":35,"match":true} +{"name":"rallies_portugal","intent":"SEARCH_RALLIES","online_total":7,"offline_total":7,"match":true} +{"name":"rallies_year_2026","intent":"SEARCH_RALLIES","online_total":54,"offline_total":54,"match":true} +{"name":"rally_aluksne_byid","intent":"SEARCH_RALLIES","online_total":1,"offline_total":1,"match":true} +{"name":"driver_rallies_freeman","intent":"SEARCH_DRIVER_RALLIES","online_total":9,"offline_total":9,"match":true} +{"name":"driver_wins_moffett","intent":"SEARCH_DRIVER_WINS","online_total":1,"offline_total":1,"match":true} +{"name":"rally_results_aluksne","intent":"GET_RALLY_RESULTS","online_total":1,"offline_total":1,"match":true} +{"name":"rally_results_donegal","intent":"GET_RALLY_RESULTS","online_total":1,"offline_total":1,"match":true} +{"name":"top_finishers_aluksne","intent":"GET_RALLY_TOP_FINISHERS","online_total":81,"offline_total":81,"match":true} +{"name":"top_finishers_donegal","intent":"GET_RALLY_TOP_FINISHERS","online_total":164,"offline_total":164,"match":true} +{"name":"top_uploaders_global","intent":"GET_TOP_UPLOADERS","online_total":259,"offline_total":259,"match":true} +{"name":"top_drivers_by_wins","intent":"GET_TOP_DRIVERS_BY_WINS","online_total":5,"offline_total":5,"match":true} +{"name":"driver_videos_freeman","intent":"SEARCH_DRIVER_VIDEOS","online_total":53,"offline_total":53,"match":true} +{"name":"video_actions_jumps","intent":"SEARCH_VIDEO_ACTIONS","online_total":3077,"offline_total":3077,"match":true} +{"name":"video_actions_crashes","intent":"SEARCH_VIDEO_ACTIONS","online_total":1627,"offline_total":1627,"match":true} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T23-35-52-240892Z/offline_failure_analysis.json b/backend/benchmarks/results/offline_search_2026-09-03T23-35-52-240892Z/offline_failure_analysis.json new file mode 100644 index 0000000..db8a271 --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T23-35-52-240892Z/offline_failure_analysis.json @@ -0,0 +1,4 @@ +{ + "wrong_confident": 0, + "failures": [] +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T23-35-52-240892Z/offline_parser_results.jsonl b/backend/benchmarks/results/offline_search_2026-09-03T23-35-52-240892Z/offline_parser_results.jsonl new file mode 100644 index 0000000..1308dba --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T23-35-52-240892Z/offline_parser_results.jsonl @@ -0,0 +1,41 @@ +{"query":"rallies in ireland in 2025","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","countries":["ireland"],"country":"ireland","years":[2025],"year":2025,"driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"what rallies happened in ireland","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","countries":["ireland"],"country":"ireland","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"portugal rallies 2024","category":"multi_filter","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","countries":["portugal"],"country":"portugal","years":[2024],"year":2024,"driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rally aluksne","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rally aluqsne","category":"typo","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rallies between 2024 and 2026","category":"multi_filter","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_RALLIES","actual_intent":"SEARCH_RALLIES","actual_fields":{"intent":"SEARCH_RALLIES","yearFrom":2024,"yearTo":2026,"driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"max freeman rallies","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_RALLIES","actual_intent":"SEARCH_DRIVER_RALLIES","actual_fields":{"intent":"SEARCH_DRIVER_RALLIES","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"what rallies did max freeman compete in","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_RALLIES","actual_intent":"SEARCH_DRIVER_RALLIES","actual_fields":{"intent":"SEARCH_DRIVER_RALLIES","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"max freemn rallies","category":"typo","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_RALLIES","actual_intent":"SEARCH_DRIVER_RALLIES","actual_fields":{"intent":"SEARCH_DRIVER_RALLIES","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rallies won by max freeman","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_WINS","actual_intent":"SEARCH_DRIVER_WINS","actual_fields":{"intent":"SEARCH_DRIVER_WINS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"max freeman wins","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_WINS","actual_intent":"SEARCH_DRIVER_WINS","actual_fields":{"intent":"SEARCH_DRIVER_WINS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"who won rally aluksne","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_RESULTS","actual_intent":"GET_RALLY_RESULTS","actual_fields":{"intent":"GET_RALLY_RESULTS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"winner of rally aluksne","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_RESULTS","actual_intent":"GET_RALLY_RESULTS","actual_fields":{"intent":"GET_RALLY_RESULTS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"results for rally aluksne","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_TOP_FINISHERS","actual_intent":"GET_RALLY_TOP_FINISHERS","actual_fields":{"intent":"GET_RALLY_TOP_FINISHERS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"leaderboard for rally aluksne","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_RALLY_TOP_FINISHERS","actual_intent":"GET_RALLY_TOP_FINISHERS","actual_fields":{"intent":"GET_RALLY_TOP_FINISHERS","rallyNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"rallyName":"0cea6942-72e3-4257-a8c1-0f8148747d82","eventNames":["0cea6942-72e3-4257-a8c1-0f8148747d82"],"eventName":"0cea6942-72e3-4257-a8c1-0f8148747d82","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"jumps from rally ireland","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_VIDEO_ACTIONS","actual_intent":"SEARCH_VIDEO_ACTIONS","actual_fields":{"intent":"SEARCH_VIDEO_ACTIONS","countries":["ireland"],"country":"ireland","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"show me crashes","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_VIDEO_ACTIONS","actual_intent":"SEARCH_VIDEO_ACTIONS","actual_fields":{"intent":"SEARCH_VIDEO_ACTIONS","actionTypes":["crash"],"actionType":"crash","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"drifts of max freeman","category":"multi_filter","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_VIDEO_ACTIONS","actual_intent":"SEARCH_VIDEO_ACTIONS","actual_fields":{"intent":"SEARCH_VIDEO_ACTIONS","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","actionTypes":["drift"],"actionType":"drift","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"videos of max freeman","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_VIDEOS","actual_intent":"SEARCH_DRIVER_VIDEOS","actual_fields":{"intent":"SEARCH_DRIVER_VIDEOS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"watch max freeman onboard","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"SEARCH_DRIVER_VIDEOS","actual_intent":"SEARCH_DRIVER_VIDEOS","actual_fields":{"intent":"SEARCH_DRIVER_VIDEOS","driverNames":["Max Freeman"],"driverName":"Max Freeman","driverIds":["7a633b52-950e-49ef-8cab-34cd43e99366"],"driverId":"7a633b52-950e-49ef-8cab-34cd43e99366","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"top uploaders","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_UPLOADERS","actual_intent":"GET_TOP_UPLOADERS","actual_fields":{"intent":"GET_TOP_UPLOADERS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"who uploaded the most videos","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_UPLOADERS","actual_intent":"GET_TOP_UPLOADERS","actual_fields":{"intent":"GET_TOP_UPLOADERS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"top drivers by wins","category":"canonical","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_DRIVERS_BY_WINS","actual_intent":"GET_TOP_DRIVERS_BY_WINS","actual_fields":{"intent":"GET_TOP_DRIVERS_BY_WINS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"most successful drivers","category":"conversational","expected_kind":"results","actual_kind":"results","expected_intent":"GET_TOP_DRIVERS_BY_WINS","actual_intent":"GET_TOP_DRIVERS_BY_WINS","actual_fields":{"intent":"GET_TOP_DRIVERS_BY_WINS","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0}} +{"query":"rally donegal","category":"ambiguity","expected_kind":"clarification","actual_kind":"clarification","expected_intent":null,"actual_intent":"SEARCH_RALLIES","actual_fields":null} +{"query":"who won rally donegal","category":"ambiguity","expected_kind":"clarification","actual_kind":"clarification","expected_intent":null,"actual_intent":"GET_RALLY_RESULTS","actual_fields":null} +{"query":"results for rally donegal","category":"ambiguity","expected_kind":"clarification","actual_kind":"clarification","expected_intent":null,"actual_intent":"GET_RALLY_TOP_FINISHERS","actual_fields":null} +{"query":"explain the offside rule in football","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what is the meaning of life","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"book me a flight to tokyo","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"rally zzzxqywv","category":"unsupported","expected_kind":"noMatch","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"compare aerodynamics of wrc cars in detail","category":"unsupported","expected_kind":"unsupported","actual_kind":"noMatch","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what is the weather","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"hello","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"thanks","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"who are you","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what can you do","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"tell me a joke","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"are you alive","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"who is the best rally driver of all time","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} +{"query":"what is the capital of france","category":"special","expected_kind":"special","actual_kind":"special","expected_intent":null,"actual_intent":null,"actual_fields":null} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T23-35-52-240892Z/offline_search_report.md b/backend/benchmarks/results/offline_search_2026-09-03T23-35-52-240892Z/offline_search_report.md new file mode 100644 index 0000000..0d88538 --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T23-35-52-240892Z/offline_search_report.md @@ -0,0 +1,29 @@ +# Offline Search Benchmark Report + +Generated: 2026-09-03T23-35-52-240892Z + +## Primary safety gate +- **wrong_confident: 0** (gate: must be 0) + +## Parser metrics +- Corpus size: 41 +- Intent accuracy: 100.0% (24/24) +- Field F1: 1.000 +- Entity resolution accuracy: 100.0% (14/14) +- Clarification accuracy: 100.0% (3/3) +- Safe unsupported rate: 100.0% (5/5) +- Special-query accuracy: 100.0% (9/9) +- OFFLINE_COVERAGE_RATE: 88.9% (24/27 answerable produced results) + +## Execution parity (offline SQLite vs online MySQL oracle) +- Cases matched: 16/16 + +## Connectivity fallback +- ONLINE: null +- OFFLINE: null +- TIMEOUT: null +- BACKEND_ERROR: null + +## Voice offline +- Cloud voice offline: NO +- On-device voice offline: DEVICE_DEPENDENT diff --git a/backend/benchmarks/results/offline_search_2026-09-03T23-35-52-240892Z/snapshot_validation.json b/backend/benchmarks/results/offline_search_2026-09-03T23-35-52-240892Z/snapshot_validation.json new file mode 100644 index 0000000..0365224 --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T23-35-52-240892Z/snapshot_validation.json @@ -0,0 +1,14 @@ +{ + "snapshot_id": "1-4b19d6e0547dcab0-full", + "table_counts": { + "rallies": 111, + "people": 8750, + "stages": 1025, + "participation": 9967, + "final_results": 326, + "driver_wins": 5, + "uploader_stats": 259, + "video_meta": 18510, + "video_actions": 32497 + } +} \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T23-35-52-240892Z/special_query_results.json b/backend/benchmarks/results/offline_search_2026-09-03T23-35-52-240892Z/special_query_results.json new file mode 100644 index 0000000..279124a --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T23-35-52-240892Z/special_query_results.json @@ -0,0 +1,47 @@ +[ + { + "query": "what is the weather", + "category": "weather", + "intercepted": true + }, + { + "query": "hello", + "category": "greeting", + "intercepted": true + }, + { + "query": "thanks", + "category": "thanks", + "intercepted": true + }, + { + "query": "who are you", + "category": "identity", + "intercepted": true + }, + { + "query": "what can you do", + "category": "capabilities", + "intercepted": true + }, + { + "query": "tell me a joke", + "category": "joke", + "intercepted": true + }, + { + "query": "are you alive", + "category": "alive", + "intercepted": true + }, + { + "query": "who is the best rally driver of all time", + "category": "rallyOpinion", + "intercepted": true + }, + { + "query": "what is the capital of france", + "category": "unsupported", + "intercepted": true + } +] \ No newline at end of file diff --git a/backend/benchmarks/results/offline_search_2026-09-03T23-35-52-240892Z/voice_offline_results.json b/backend/benchmarks/results/offline_search_2026-09-03T23-35-52-240892Z/voice_offline_results.json new file mode 100644 index 0000000..b99b8cd --- /dev/null +++ b/backend/benchmarks/results/offline_search_2026-09-03T23-35-52-240892Z/voice_offline_results.json @@ -0,0 +1,6 @@ +{ + "cloud_voice_offline": "NO (network required)", + "on_device_voice_offline": "DEVICE_DEPENDENT (only where OS on-device recognizer supports it)", + "auto_submit": false, + "transcript_editable": true +} \ No newline at end of file diff --git a/backend/tests/unit/test_latency_instrumentation.py b/backend/tests/unit/test_latency_instrumentation.py new file mode 100644 index 0000000..7771ac6 --- /dev/null +++ b/backend/tests/unit/test_latency_instrumentation.py @@ -0,0 +1,257 @@ +"""Backend latency instrumentation. + +Covers the three properties the timing record has to hold: it measures the +whole request (not just the part inside the handler), it correlates end to end +with the client's id, and it never carries user content. +""" + +import json +import logging + +import pytest +from httpx import ASGITransport, AsyncClient + +from app.api.v1.conversation import get_conversational_service +from app.domain.conversation_session import SearchConversationSession +from app.domain.results import SearchResponse +from app.domain.search_intent import SearchIntent +from app.domain.search_plan import INTENT_TO_STRATEGY, SearchPlan +from app.main import app +from app.observability import Phase, RequestTimings, request_scope +from app.observability.logging import JsonFormatter, log_request_timing, sanitize +from app.observability.middleware import REQUEST_ID_HEADER, SERVER_TIMING_HEADER +from app.services.conversational_search_service import ( + ConversationalSearchResult, + ConversationalSearchService, +) + +pytestmark = pytest.mark.unit + + +class _StubService: + """Stands in for the real pipeline, recording phase times the way it does.""" + + def __init__(self, *, clarification: bool = False) -> None: + self.clarification = clarification + + async def search(self, query, *, session=None, language=None, expose_timings=False, **_): + from app.observability import current_request_id, current_timings + + timings = current_timings() + if timings is not None: + timings.add(Phase.QUERY_UNDERSTANDING, 12.0) + timings.add(Phase.GEMINI, 11.0) + timings.add(Phase.REPOSITORY_DB, 5.0) + timings.update(model="gemini-3.5-flash-lite", gemini_called=True) + result = ConversationalSearchResult( + requires_clarification=self.clarification, + clarification_question="Which one?" if self.clarification else None, + search_response=None + if self.clarification + else SearchResponse( + intent=SearchIntent.SEARCH_RALLIES, + results=[], + total_count=0, + has_more=False, + limit=20, + offset=0, + ), + search_plan=None + if self.clarification + else SearchPlan( + intent=SearchIntent.SEARCH_RALLIES, + strategy=INTENT_TO_STRATEGY[SearchIntent.SEARCH_RALLIES], + limit=20, + offset=0, + ), + request_id=current_request_id(), + timings=timings.snapshot() if (expose_timings and timings) else None, + ) + return (session or SearchConversationSession()), result + + +def _override(service): + app.dependency_overrides[get_conversational_service] = lambda: service + + +async def _post(payload, headers=None): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: + return await client.post("/v1/conversation/search", json=payload, headers=headers or {}) + + +async def test_client_request_id_propagates_into_response_and_logs(caplog): + _override(_StubService()) + try: + with caplog.at_level(logging.INFO, logger="app.latency"): + response = await _post({"query": "rallies in ireland"}, {REQUEST_ID_HEADER: "trace-abc-1"}) + finally: + app.dependency_overrides.clear() + + assert response.status_code == 200 + # Echoed on the wire in both the header and the body. + assert response.headers[REQUEST_ID_HEADER] == "trace-abc-1" + assert response.json()["traceId"] == "trace-abc-1" + # And present on the structured timing line, which is what makes a client + # record joinable to a backend record. + record = next(r for r in caplog.records if r.getMessage() == "search_timing") + assert record.timing["request_id"] == "trace-abc-1" + + +async def test_a_missing_or_unsafe_client_id_is_replaced_not_echoed(): + _override(_StubService()) + try: + no_header = await _post({"query": "rallies in ireland"}) + unsafe = await _post( + {"query": "rallies in ireland"}, + {REQUEST_ID_HEADER: "drop table users; -- " + "x" * 200}, + ) + finally: + app.dependency_overrides.clear() + + generated = no_header.headers[REQUEST_ID_HEADER] + assert len(generated) == 32 and generated.isalnum() + assert "drop table" not in unsafe.headers[REQUEST_ID_HEADER] + + +async def test_timing_record_covers_every_required_phase(caplog): + _override(_StubService()) + try: + with caplog.at_level(logging.INFO, logger="app.latency"): + await _post({"query": "rallies in ireland"}, {REQUEST_ID_HEADER: "trace-phases"}) + finally: + app.dependency_overrides.clear() + + timing = next( + r.timing for r in caplog.records if r.getMessage() == "search_timing" + ) + for key in ( + "request_id", + "total_backend_ms", + "dependencies_ms", + "query_understanding_ms", + "gemini_ms", + "repository_db_ms", + "serialization_ms", + "model", + "gemini_called", + "intent", + "clarification", + "status_code", + ): + assert key in timing, key + assert timing["gemini_called"] is True + assert timing["clarification"] is False + + +async def test_clarification_is_reported_as_such(caplog): + _override(_StubService(clarification=True)) + try: + with caplog.at_level(logging.INFO, logger="app.latency"): + await _post({"query": "donegal"}, {REQUEST_ID_HEADER: "trace-clarify"}) + finally: + app.dependency_overrides.clear() + + timing = next(r.timing for r in caplog.records if r.getMessage() == "search_timing") + assert timing["clarification"] is True + assert timing["intent"] is None + + +async def test_total_backend_ms_includes_time_outside_the_handler(): + """The regression this instrumentation exists to prevent. + + The pre-existing `totalLatencyMs` started inside the service, after FastAPI + had already resolved dependencies (where the DB connection is opened). A + cold request measured 2835 ms at the client while reporting 1024 ms. The + header total must account for the dependency phase too. + """ + import asyncio + + class _SlowDependency(_StubService): + pass + + async def slow_service(): + await asyncio.sleep(0.05) + return _SlowDependency() + + app.dependency_overrides[get_conversational_service] = slow_service + try: + response = await _post({"query": "rallies in ireland"}, {REQUEST_ID_HEADER: "trace-slow"}) + finally: + app.dependency_overrides.clear() + + total = float(response.headers[SERVER_TIMING_HEADER]) + assert total >= 50.0, f"dependency time missing from the total: {total}" + + +def test_timings_accumulate_across_repeated_phases(): + timings = RequestTimings(request_id="t") + timings.add(Phase.GEMINI, 100.0) + timings.add(Phase.GEMINI, 250.0) + # A provider retry must show the summed cost, not just the last attempt. + assert timings.snapshot()["gemini_ms"] == 350.0 + + +def test_user_content_is_stripped_from_timing_records(): + record = sanitize( + { + "request_id": "t", + "query": "who won rally donegal", + "transcript": "spoken words", + "api_key": "secret", + "session": {"history": ["..."]}, + "gemini_ms": 12.0, + } + ) + assert record == {"request_id": "t", "gemini_ms": 12.0} + + +def test_json_formatter_emits_one_parseable_object_per_line(): + record = logging.LogRecord("app.latency", logging.INFO, __file__, 1, "search_timing", None, None) + record.timing = {"request_id": "t", "total_backend_ms": 1.5} + line = JsonFormatter().format(record) + assert "\n" not in line + assert json.loads(line)["request_id"] == "t" + + +def test_request_scope_isolates_and_restores_context(): + from app.observability import current_request_id, current_timings + + assert current_request_id() is None + with request_scope("outer"): + assert current_request_id() == "outer" + with request_scope("inner"): + assert current_request_id() == "inner" + assert current_request_id() == "outer" + assert current_timings() is not None + assert current_request_id() is None + assert current_timings() is None + + +def test_recording_a_phase_outside_a_request_scope_is_a_no_op(): + from app.observability import current_timings + + # Pipeline code runs in benchmarks and scripts with no scope active; it must + # simply record nothing rather than fail. + assert current_timings() is None + + +async def test_search_result_carries_debug_timings_only_when_enabled(): + from app.query_understanding.provider import ProviderConfig + from app.query_understanding.providers import MockProvider + from app.query_understanding.service import QueryUnderstandingService + + service = ConversationalSearchService( + query_parser=QueryUnderstandingService( + MockProvider(ProviderConfig(provider="mock", model="mock-parser-v1")) + ), + ) + with request_scope("trace-off"): + _, off = await service.search("rallies in ireland", expose_timings=False) + with request_scope("trace-on"): + _, on = await service.search("rallies in ireland", expose_timings=True) + + assert off.request_id == "trace-off" + assert off.timings is None + assert on.timings is not None + assert on.timings["request_id"] == "trace-on" + assert "query_understanding_ms" in on.timings diff --git a/bin/run_live_voice_eval.dart b/bin/run_live_voice_eval.dart deleted file mode 100644 index 9288c6b..0000000 --- a/bin/run_live_voice_eval.dart +++ /dev/null @@ -1,186 +0,0 @@ -import 'dart:io'; -import 'package:ai_rally_search/models/supported_language.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/database_entity_resolver.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/entity_lookup_repository.dart'; -import 'package:ai_rally_search/services/llm/llm_query_parser_factory.dart'; -import 'package:ai_rally_search/services/llm/natural_language_search_service.dart'; -import 'package:ai_rally_search/services/search_repository.dart'; -import 'package:ai_rally_search/services/speech/openai_speech_to_text_service.dart'; -import 'package:ai_rally_search/services/speech/speech_config.dart'; -import '../test/eval/audio_asset_resolver.dart'; -import '../test/eval/live_voice_benchmark_evaluator.dart'; -import '../test/eval/manifest/benchmark_manifest.dart'; -import '../test/eval/manifest/human_benchmark_models.dart'; -import '../test/eval/manifest/human_pilot_manifest.dart'; - -Map _loadDotEnv() { - final env = {}; - final file = File('.env'); - if (file.existsSync()) { - for (final line in file.readAsLinesSync()) { - final trimmed = line.trim(); - if (trimmed.isEmpty || trimmed.startsWith('#')) continue; - final eqIdx = trimmed.indexOf('='); - if (eqIdx > 0) { - final key = trimmed.substring(0, eqIdx).trim(); - var val = trimmed.substring(eqIdx + 1).trim(); - if (val.startsWith('"') && val.endsWith('"') && val.length >= 2) { - val = val.substring(1, val.length - 1); - } else if (val.startsWith("'") && val.endsWith("'") && val.length >= 2) { - val = val.substring(1, val.length - 1); - } - env[key] = val; - } - } - } - return env; -} - -void main(List args) async { - if (args.contains('--help') || args.contains('-h')) { - stdout.writeln('AI Rally Voice Search Benchmark Runner'); - stdout.writeln('========================================'); - stdout.writeln('Usage: dart run bin/run_live_voice_eval.dart [options]'); - stdout.writeln(); - stdout.writeln('Options:'); - stdout.writeln(' --dataset= Target benchmark dataset (default: synthetic)'); - stdout.writeln(' --language= Filter by ISO language code (e.g. en, ga, ur, de)'); - stdout.writeln(' --archetype= Filter by semantic archetype letter (A, B, C, D, or E)'); - stdout.writeln(' --sample= Run evaluation on a single sample ID'); - stdout.writeln(' --model= Speech-to-text model override (default: whisper-1)'); - stdout.writeln(' --outputDir= Reports directory (default: test/eval/reports)'); - stdout.writeln(' --help, -h Show this help message'); - exit(0); - } - - // Parse arguments - String datasetType = 'synthetic'; - String? languageFilter; - String? archetypeFilter; - String? sampleFilter; - String outputDir = 'test/eval/reports'; - - for (final arg in args) { - if (arg.startsWith('--dataset=')) { - datasetType = arg.substring('--dataset='.length).trim().toLowerCase(); - } else if (arg.startsWith('--language=')) { - languageFilter = arg.substring('--language='.length).trim().toLowerCase(); - } else if (arg.startsWith('--archetype=')) { - archetypeFilter = arg.substring('--archetype='.length).trim().toUpperCase(); - } else if (arg.startsWith('--sample=')) { - sampleFilter = arg.substring('--sample='.length).trim(); - } else if (arg.startsWith('--outputDir=')) { - outputDir = arg.substring('--outputDir='.length).trim(); - } - } - - final isHumanDataset = datasetType == 'human'; - final benchmarkType = isHumanDataset ? BenchmarkType.human : BenchmarkType.synthetic; - - stdout.writeln('==========================================================='); - stdout.writeln('🎙️ Live Voice Search Benchmark Runner [${benchmarkType.name.toUpperCase()}]'); - stdout.writeln('==========================================================='); - if (isHumanDataset) { - stdout.writeln('⚠️ WAVE-1 HUMAN PILOT: Failure-discovery baseline across native speakers.'); - stdout.writeln('⚠️ Human and synthetic metrics remain strictly segregated.'); - } - - final env = _loadDotEnv(); - final apiKey = env['OPENAI_API_KEY'] ?? Platform.environment['OPENAI_API_KEY']; - if (apiKey == null || apiKey.isEmpty) { - stderr.writeln('ERROR: OPENAI_API_KEY must be configured in .env or environment'); - exit(1); - } - - final speechModel = env['SPEECH_MODEL'] ?? Platform.environment['SPEECH_MODEL'] ?? 'whisper-1'; - final speechConfig = SpeechConfig( - providerType: SpeechProviderType.openAiDirectDev, - endpointUrl: 'https://api.openai.com/v1/audio/transcriptions', - apiKey: apiKey, - model: speechModel, - ); - - final speechService = OpenAiSpeechToTextService(config: speechConfig); - final parser = LlmQueryParserFactory.create(); - final lookupRepo = DatabaseEntityLookupRepository(); - final resolver = DatabaseEntityResolver(repository: lookupRepo); - final searchRepo = SearchRepository(); - - final nlSearchService = NaturalLanguageSearchService( - parser: parser, - entityResolver: resolver, - repository: searchRepo, - ); - - final assetResolver = isHumanDataset - ? const LocalDirectoryAssetResolver(Directory('test/eval/audio/human')) - : null; - - final evaluator = LiveVoiceBenchmarkEvaluator( - speechService: speechService, - nlSearchService: nlSearchService, - assetResolver: assetResolver, - ); - - // Select Manifest - List entries; - if (isHumanDataset) { - entries = List.from(HumanPilotBenchmarkManifest.entries); - } else { - entries = List.from(SyntheticSmokeBenchmarkManifest.entries); - } - - // Apply Filters - if (languageFilter != null) { - entries = entries.where((e) => e.language.languageCode == languageFilter).toList(); - } - if (archetypeFilter != null) { - entries = entries.where((e) { - if (e is HumanBenchmarkManifestEntry) { - return e.archetype.code == archetypeFilter; - } - return true; - }).toList(); - } - if (sampleFilter != null) { - entries = entries.where((e) => e.id == sampleFilter).toList(); - } - - if (entries.isEmpty) { - stdout.writeln('No benchmark entries matched filters (language: $languageFilter, archetype: $archetypeFilter, sample: $sampleFilter).'); - exit(0); - } - - stdout.writeln('Executing evaluation on ${entries.length} samples...\n'); - - final results = await evaluator.evaluateManifest( - entries, - onProgress: (sample, index, total) { - final status = sample.audioMissing - ? '⚠️ MISSING' - : (sample.searchSemanticSuccess ? '✅ SUCCESS' : '❌ FAILED'); - final langStr = '${sample.entry.language.displayName} (${sample.entry.locale})'; - stdout.writeln( - '[$index/$total] $status $langStr [${sample.entry.id}] ' - '| Transcript: "${sample.actualTranscript}" ' - '| WER: ${(sample.wer * 100).toStringAsFixed(1)}% ' - '| Intent: ${sample.intentMatched} ' - '| Attribution: ${sample.failureAttribution.label} ' - '| E2E: ${sample.totalLatencyMs}ms', - ); - }, - ); - - final mdPath = await LiveVoiceBenchmarkEvaluator.generateReports( - results: results, - outputDir: outputDir, - benchmarkType: benchmarkType, - modelName: speechModel, - ); - - stdout.writeln('\n==========================================================='); - stdout.writeln('📊 Benchmark evaluation complete.'); - stdout.writeln('📄 Markdown report generated: $mdPath'); - stdout.writeln('==========================================================='); - exit(0); -} diff --git a/bin/run_structured_parity.dart b/bin/run_structured_parity.dart deleted file mode 100644 index ff99268..0000000 --- a/bin/run_structured_parity.dart +++ /dev/null @@ -1,46 +0,0 @@ -import 'dart:convert'; -import 'dart:io'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:ai_rally_search/models/search_query.dart'; -import 'package:ai_rally_search/models/search_results.dart'; -import 'package:ai_rally_search/models/video_action.dart'; -import 'package:ai_rally_search/services/database_service.dart'; -import 'package:ai_rally_search/services/search_repository.dart'; - -Future main(List args) async { - if (args.length != 2) { - stderr.writeln('usage: dart run bin/run_structured_parity.dart FIXTURES OUTPUT'); - exitCode = 64; return; - } - await dotenv.load(fileName: '.env'); - final document=jsonDecode(await File(args[0]).readAsString()) as Map; - if (document['schemaVersion'] != '1.0') throw StateError('Unsupported fixture schema'); - final sink=File(args[1]).openWrite(); - final db=DatabaseService(); final repo=SearchRepository(dbService: db); - try { - for (final raw in document['cases'] as List) { - final fixture=Map.from(raw as Map); - final query=SearchQuery.fromJson(Map.from(fixture['searchQuery'] as Map)); - final response=await repo.search(query); - final ids=response.results.map(_canonicalId).toList(); - sink.writeln(jsonEncode({ - 'schemaVersion':'1.0','runtime':'dart','caseId':fixture['caseId'], - 'searchQuery':fixture['searchQuery'],'intent':response.intent.toIntentString(), - 'orderedCanonicalIds':ids,'total':response.totalCount,'limit':response.limit, - 'offset':response.offset,'hasMore':response.hasMore, - 'currentPage':(response.offset ~/ response.limit)+1, - })); - } - } finally { await sink.flush(); await sink.close(); await db.close(); } -} - -String _canonicalId(dynamic item) { - if (item is RallySearchResult) return item.eventId; - if (item is RallyParticipationResult) return item.rallyId; - if (item is RallyResult) return item.id.toString(); - if (item is VideoAction) return item.id.toString(); - if (item is VideoSearchResult) return item.videoId.toString(); - if (item is UploaderSearchResult) return item.uploaderId; - if (item is DriverWinResult) return item.driverId ?? item.driverName; - throw StateError('No canonical identity for ${item.runtimeType}'); -} diff --git a/bin/run_voice_eval.dart b/bin/run_voice_eval.dart deleted file mode 100644 index 36687b5..0000000 --- a/bin/run_voice_eval.dart +++ /dev/null @@ -1,67 +0,0 @@ -import 'dart:io'; -import 'package:ai_rally_search/services/llm/entity_resolution/database_entity_resolver.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/entity_lookup_repository.dart'; -import 'package:ai_rally_search/services/llm/llm_provider_config.dart'; -import 'package:ai_rally_search/services/llm/llm_query_parser_factory.dart'; -import 'package:ai_rally_search/services/llm/natural_language_search_service.dart'; -import 'package:ai_rally_search/services/search_repository.dart'; -import 'package:ai_rally_search/services/speech/speech_config.dart'; -import 'package:ai_rally_search/services/speech/speech_service_factory.dart'; -import '../test/eval/multilingual_voice_benchmark_cases.dart'; -import '../test/eval/voice_benchmark_evaluator.dart'; - -void main(List args) async { - stdout.writeln('==========================================================='); - stdout.writeln('🎙️ Phase 5B Multilingual Voice Search Benchmark Runner'); - stdout.writeln('==========================================================='); - - final speechConfig = SpeechConfig.fromEnvironment(); - stdout.writeln('STT Provider: ${speechConfig.providerType.name}'); - stdout.writeln('STT Endpoint: ${speechConfig.endpointUrl}'); - - final speechService = SpeechServiceFactory.create(config: speechConfig); - final searchRepo = SearchRepository(); - final parser = LlmQueryParserFactory.create(); - final lookupRepo = DatabaseEntityLookupRepository(); - final resolver = DatabaseEntityResolver(repository: lookupRepo); - - final nlSearchService = NaturalLanguageSearchService( - parser: parser, - entityResolver: resolver, - repository: searchRepo, - ); - - final evaluator = VoiceBenchmarkEvaluator( - speechService: speechService, - nlSearchService: nlSearchService, - searchRepository: searchRepo, - ); - - stdout.writeln('Running benchmark across ${MultilingualVoiceBenchmarkCases.all.length} multilingual test cases...'); - final results = await evaluator.evaluateSuite(MultilingualVoiceBenchmarkCases.all); - - const outputDir = 'test/eval/reports'; - VoiceBenchmarkEvaluator.saveEvaluationReports( - results: results, - outputDir: outputDir, - ); - - stdout.writeln('\nBenchmark completed successfully!'); - stdout.writeln('Reports saved in $outputDir/'); - - final avgWer = results.map((r) => r.wordErrorRate).reduce((a, b) => a + b) / results.length; - final driverPres = results.where((r) => r.driverPreserved).length / results.length; - final rallyPres = results.where((r) => r.rallyPreserved).length / results.length; - final actionPres = results.where((r) => r.actionPreserved).length / results.length; - final avgLatency = results.map((r) => r.totalLatencyMs).reduce((a, b) => a + b) / results.length; - - stdout.writeln('\n---------------- Summary ----------------'); - stdout.writeln('Average WER: ${(avgWer * 100).toStringAsFixed(1)}%'); - stdout.writeln('Driver Name Preservation: ${(driverPres * 100).toStringAsFixed(1)}%'); - stdout.writeln('Rally Name Preservation: ${(rallyPres * 100).toStringAsFixed(1)}%'); - stdout.writeln('Action Term Preservation: ${(actionPres * 100).toStringAsFixed(1)}%'); - stdout.writeln('Average E2E Latency: ${avgLatency.toStringAsFixed(0)} ms'); - stdout.writeln('-----------------------------------------'); - - exit(0); -} diff --git a/lib/main.dart b/lib/main.dart index 70f8a3b..3d0bacf 100644 --- a/lib/main.dart +++ b/lib/main.dart @@ -6,7 +6,9 @@ import 'services/offline/offline_bootstrap.dart'; Future main() async { WidgetsFlutterBinding.ensureInitialized(); - await dotenv.load(fileName: '.env'); + // Loads PUBLIC client config only (API base URL, non-secret speech settings). + // Server secrets (DB credentials, API keys) are never bundled in the app. + await dotenv.load(fileName: 'assets/config/app_config.env'); // Initialise the offline search stack (local SQLite snapshot + sync). Fully // guarded: on any failure this returns null and the app runs online-only. diff --git a/lib/models/rally_stream.dart b/lib/models/rally_stream.dart deleted file mode 100644 index 5e9c106..0000000 --- a/lib/models/rally_stream.dart +++ /dev/null @@ -1,103 +0,0 @@ -class RallyStream { - final int id; - final int? videoId; - final double? clipDuration; - final String? videoType; - final String? onDemandUrl; - final DateTime? createdAt; - final DateTime? updatedAt; - final double? clipStartTime; - final String? clipStatus; - final int downloadCounter; - final int shareCounter; - - const RallyStream({ - required this.id, - this.videoId, - this.clipDuration, - this.videoType, - this.onDemandUrl, - this.createdAt, - this.updatedAt, - this.clipStartTime, - this.clipStatus, - this.downloadCounter = 0, - this.shareCounter = 0, - }); - - factory RallyStream.fromMap(Map map) { - return RallyStream( - id: _parseInt(map['id']) ?? 0, - videoId: _parseInt(map['video_id']), - clipDuration: _parseDouble(map['clip_duration']), - videoType: map['video_type']?.toString(), - onDemandUrl: map['on_demand_url']?.toString(), - createdAt: _parseDateTime(map['created_at']), - updatedAt: _parseDateTime(map['updated_at']), - clipStartTime: _parseDouble(map['clip_start_time']), - clipStatus: map['clip_status']?.toString(), - downloadCounter: _parseInt(map['download_counter']) ?? 0, - shareCounter: _parseInt(map['share_counter']) ?? 0, - ); - } - - static int? _parseInt(dynamic value) { - if (value == null) return null; - if (value is int) return value; - return int.tryParse(value.toString()); - } - - static double? _parseDouble(dynamic value) { - if (value == null) return null; - if (value is double) return value; - if (value is int) return value.toDouble(); - return double.tryParse(value.toString()); - } - - static DateTime? _parseDateTime(dynamic value) { - if (value == null) return null; - if (value is DateTime) return value; - return DateTime.tryParse(value.toString()); - } - - String get formattedDuration { - if (clipDuration == null) return '0.0s'; - if (clipDuration! < 60) { - return '${clipDuration!.toStringAsFixed(1)}s'; - } - final minutes = (clipDuration! / 60).floor(); - final remainingSecs = (clipDuration! % 60).toStringAsFixed(1); - return '${minutes}m ${remainingSecs}s'; - } - - String get formattedClipRange { - final start = clipStartTime ?? 0.0; - if (clipDuration != null && clipDuration! > 0) { - final end = start + clipDuration!; - return '${_formatTimeSeconds(start)} → ${_formatTimeSeconds(end)}'; - } - return '${_formatTimeSeconds(start)} → End'; - } - - static String _formatTimeSeconds(double totalSeconds) { - final totalSecsInt = totalSeconds.floor(); - final hours = (totalSecsInt / 3600).floor(); - final minutes = ((totalSecsInt % 3600) / 60).floor().toString().padLeft(2, '0'); - final seconds = (totalSecsInt % 60).toString().padLeft(2, '0'); - if (hours > 0) { - return '$hours:$minutes:$seconds'; - } - return '$minutes:$seconds'; - } - - String get formattedDate { - if (createdAt == null) return 'Unknown date'; - final y = createdAt!.year.toString().padLeft(4, '0'); - final m = createdAt!.month.toString().padLeft(2, '0'); - final d = createdAt!.day.toString().padLeft(2, '0'); - final h = createdAt!.hour.toString().padLeft(2, '0'); - final min = createdAt!.minute.toString().padLeft(2, '0'); - final s = createdAt!.second.toString().padLeft(2, '0'); - return '$y-$m-$d $h:$min:$s'; - } -} diff --git a/lib/screens/general_search_screen.dart b/lib/screens/general_search_screen.dart index 61aa7e7..3aec63a 100644 --- a/lib/screens/general_search_screen.dart +++ b/lib/screens/general_search_screen.dart @@ -1,5 +1,6 @@ import 'dart:async'; +import 'package:clock/clock.dart'; import 'package:flutter/material.dart'; import '../l10n/generated/app_localizations.dart'; @@ -22,7 +23,9 @@ import '../services/python_search_api_client.dart'; import '../services/friendly_response_service.dart'; import '../services/offline/offline_messaging.dart'; import '../services/offline/offline_search_engine.dart'; -import '../services/offline/offline_search_router.dart'; +import '../services/latency/latency_policy.dart'; +import '../services/latency/search_latency_coordinator.dart'; +import '../services/latency/search_telemetry.dart'; import '../services/offline/offline_snapshot_sync.dart'; import '../widgets/offline_banner.dart'; import '../widgets/action_player_modal.dart'; @@ -42,7 +45,29 @@ import '../services/speech/speech_to_text_service.dart'; import '../services/speech/speech_service_factory.dart'; import '../theme/app_theme.dart'; import '../widgets/results_skeleton.dart'; -import 'rally_streams_page.dart'; + +/// Rally-toned loading copy for a search whose intent is not yet known. +const String _kCheckingTimingSheets = 'Checking the timing sheets...'; + +/// One completed authoritative online turn, held so it can be applied now or +/// offered for later promotion without re-issuing the request. +class _OnlineTurn { + final NaturalLanguageSearchResult result; + final SearchConversationSession? session; + + /// The generation the backend echoed back, when it echoes one. + final int? echoedRequestId; + final int? networkRoundtripMs; + final String? transcript; + + const _OnlineTurn({ + required this.result, + this.session, + this.echoedRequestId, + this.networkRoundtripMs, + this.transcript, + }); +} /// User-facing category of a results-area failure. Presentation only — the /// underlying exception/state is unchanged; this only selects friendly copy. @@ -82,6 +107,14 @@ class GeneralSearchScreen extends StatefulWidget { final OfflineSnapshotSync? offlineSync; final ConnectivityProbe? connectivityProbe; + /// The single latency/fallback policy. Tests inject short budgets; there is + /// no second place in the app where these numbers are defined. + final LatencyPolicy latencyPolicy; + + /// Optional sink for client latency records. Defaults to discarding them, so + /// instrumentation costs nothing unless deliberately enabled. + final SearchTelemetrySink? telemetrySink; + const GeneralSearchScreen({ super.key, this.initialQuery, @@ -95,6 +128,8 @@ class GeneralSearchScreen extends StatefulWidget { this.offlineEngine, this.offlineSync, this.connectivityProbe, + this.latencyPolicy = LatencyPolicy.standard, + this.telemetrySink, }); @override @@ -165,6 +200,16 @@ class _GeneralSearchScreenState extends State { ConnectivityProbe? _connectivityProbe; OfflineUxState _offlineState = OfflineUxState.online; Duration? _offlineAge; + late final SearchLatencyCoordinator<_OnlineTurn> _coordinator; + late final SearchTelemetrySink _telemetrySink; + StreamSubscription>? _searchSubscription; + + /// A late authoritative result waiting behind an explicit "show latest" + /// action. Held, never applied on its own. + _OnlineTurn? _pendingOnlineTurn; + String? _pendingOnlineQueryText; + int? _pendingOnlineGeneration; + bool get _hasPendingOnlineResult => _pendingOnlineTurn != null; bool get _offlineActive => _offlineState != OfflineUxState.online; static const OfflineMessagingService _offlineMessaging = OfflineMessagingService(); @@ -174,6 +219,12 @@ class _GeneralSearchScreenState extends State { _offlineEngine = widget.offlineEngine; _offlineSync = widget.offlineSync; _connectivityProbe = widget.connectivityProbe; + _telemetrySink = widget.telemetrySink ?? const NullSearchTelemetrySink(); + _coordinator = SearchLatencyCoordinator<_OnlineTurn>( + connectivity: _connectivityProbe, + engine: _offlineEngine, + policy: widget.latencyPolicy, + ); final backendConfig = SearchBackendConfig.fromEnvironment(); // Python FastAPI is the sole authoritative search backend. There is no // runtime legacy switch and no silent in-app Dart fallback: when the Python @@ -213,6 +264,12 @@ class _GeneralSearchScreenState extends State { @override void dispose() { + // Stops delivery of any in-flight search event. The online request itself + // is left to complete rather than being torn down mid-flight; its result + // is simply never applied. + _searchSubscription?.cancel(); + _searchSubscription = null; + _discardPendingOnline(); _searchController.removeListener(_onSearchControllerChanged); _nativeSpeechService.dispose(); _cloudSpeechService.dispose(); @@ -265,6 +322,9 @@ class _GeneralSearchScreenState extends State { final requestSession = _session; final nextRequestId = _session.activeRequestId + 1; + // Opaque correlation id, sent as X-Request-Id and echoed by the backend + // into its structured timing line. Never contains query text. + final traceId = newRequestId(); if (_sttSource != null) { final wasEdited = _sttRawTranscript != null && _sttRawTranscript!.trim().toLowerCase() != queryText.toLowerCase(); @@ -279,12 +339,17 @@ class _GeneralSearchScreenState extends State { 'wasEditedBeforeSubmit': wasEdited, }; } + + // Dropping any offer from the previous search before the new one starts is + // what stops a stale "show latest" from promoting a result belonging to an + // abandoned query. + _discardPendingOnline(); setState(() { if (queryText.isNotEmpty) _searchController.text = queryText; _hasSearched = true; _session = _session.copyWith(activeRequestId: nextRequestId); _isLoading = true; - _loadingStatus = 'Understanding your search...'; + _loadingStatus = _kCheckingTimingSheets; _errorMessage = null; _specialMessage = null; _emptyResultsMessage = null; @@ -294,25 +359,20 @@ class _GeneralSearchScreenState extends State { _currentPage = 1; }); - // NETWORK_FIRST_WITH_LOCAL_FALLBACK: if the device is plainly offline, skip - // the online attempt entirely and answer from the local snapshot. - if (queryText.isNotEmpty && _offlineEngine != null && await _isPlainlyOffline()) { - if (!mounted || _session.activeRequestId != nextRequestId) return; - await _runOfflineFallback(queryText, OfflineUxState.offlineLocalResults); - return; - } + final dispatchedAt = clock.now(); - try { + // The authoritative online turn. Whichever transport applies, the + // coordinator sees one future; it is started immediately and is never + // cancelled by a fallback. + Future<_OnlineTurn> runOnline() async { final searchContext = SearchContext( currentYear: DateTime.now().year, locale: _selectedLanguage.localeCode, languageCode: _selectedLanguage.languageCode, - referents: _session.referents, - previousQuery: _session.activeQuery, + referents: requestSession.referents, + previousQuery: requestSession.activeQuery, ); - final NaturalLanguageSearchResult result; - SearchConversationSession? backendSession; if (_pythonApiClient != null && spokenResult?.audioContext != null) { final audio = spokenResult!.audioContext!; final response = await _pythonApiClient!.voice( @@ -322,202 +382,445 @@ class _GeneralSearchScreenState extends State { language: _selectedLanguage.languageCode, requestId: nextRequestId, editedTranscript: queryText.isEmpty ? null : queryText, + traceId: traceId, ); - if (response.requestId != null && response.requestId != nextRequestId) { - return; - } - backendSession = response.session; - result = response.result; - final transcript = response.transcription?.text.trim() ?? ''; - if (transcript.isNotEmpty) _searchController.text = transcript; - } else if (_pythonApiClient != null) { + return _OnlineTurn( + result: response.result, + session: response.session, + echoedRequestId: response.requestId, + networkRoundtripMs: response.networkRoundtripMs, + transcript: response.transcription?.text.trim(), + ); + } + if (_pythonApiClient != null) { final response = await _pythonApiClient!.conversation( query: queryText, session: requestSession, language: _selectedLanguage.languageCode, requestId: nextRequestId, + traceId: traceId, ); - if (response.requestId != null && response.requestId != nextRequestId) { - return; - } - backendSession = response.session; - result = response.result; - } else if (_nlSearchService != null) { + return _OnlineTurn( + result: response.result, + session: response.session, + echoedRequestId: response.requestId, + networkRoundtripMs: response.networkRoundtripMs, + ); + } + if (_nlSearchService != null) { // Test-only injected in-app search path. - result = spokenResult != null + final result = spokenResult != null ? await _nlSearchService.searchSpoken( spokenResult, context: searchContext, ) : await _nlSearchService.search(queryText, context: searchContext); - } else { - // No backend configured: surface a clean error instead of any legacy - // fallback. - throw const PythonApiException( - 'config', - 'Search backend is not configured (PYTHON_BACKEND_BASE_URL missing).', - ); + return _OnlineTurn(result: result); } + // No backend configured: surface a clean error instead of any legacy + // fallback. + throw const PythonApiException( + 'config', + 'Search backend is not configured (PYTHON_BACKEND_BASE_URL missing).', + ); + } - if (!mounted || _session.activeRequestId != nextRequestId) return; + // Detach from the previous search without awaiting: cancelling is only + // about stopping delivery, and awaiting it would delay dispatching the new + // query behind the old one's teardown. The generation guard, not the + // cancel, is what keeps a stale event from being applied. + unawaited(_searchSubscription?.cancel() ?? Future.value()); + _searchSubscription = _coordinator + .run( + generation: nextRequestId, + rawText: queryText, + online: runOnline, + limit: _pageSize, + ) + .listen( + (event) => unawaited( + _handleSearchEvent( + event: event, + queryText: queryText, + traceId: traceId, + dispatchedAt: dispatchedAt, + ), + ), + onDone: () { + if (_usePythonBackend && spokenResult?.audioContext != null) { + spokenResult!.disposeAudio(); + } + }, + ); + } - if (result.isSpecialResponse) { - // Easter-egg / personality response: show only the playful message. - // No DB results, no zero-results state, no clarification, no stale - // pagination. The rally conversation context (referents/active query) - // from any prior search is intentionally preserved. - setState(() { - if (backendSession != null) _session = backendSession; - _isLoading = false; - _specialMessage = result.friendlyMessage; - _searchResponse = null; - _totalCount = 0; - _clarificationQuestion = null; - _clarificationCandidates = []; - _pendingClarification = null; - _errorMessage = null; - _emptyResultsMessage = null; - }); - return; - } + /// Applies one coordinator event. + /// + /// Two guards run before anything is rendered: the widget must still be + /// mounted, and the event's generation must still be the active one. Between + /// them they cover a disposed screen and a superseded query — a late result + /// for query A can never overwrite query B. + Future _handleSearchEvent({ + required SearchEvent<_OnlineTurn> event, + required String queryText, + required String traceId, + required DateTime dispatchedAt, + }) async { + if (!mounted || _session.activeRequestId != event.generation) return; + final connectivity = event.connectivity; + + switch (event.stage) { + case SearchStage.online: + await _applyOnlineTurn(event.online!, queryText, event.generation); + _recordSearchTelemetry( + traceId: traceId, + dispatchedAt: dispatchedAt, + connectivity: connectivity, + source: SearchResultSource.online, + networkRoundtripMs: event.online!.networkRoundtripMs, + timeToFirstResultMs: event.elapsedMs, + ); - if (result.requiresClarification) { + case SearchStage.offlineImmediate: + _renderOfflineOutcome( + event.offline!, + OfflineUxState.offlineLocalResults, + event.generation, + ); + _recordSearchTelemetry( + traceId: traceId, + dispatchedAt: dispatchedAt, + connectivity: connectivity, + source: SearchResultSource.offline, + localParserCouldAnswer: true, + timeToFirstResultMs: event.elapsedMs, + ); + + case SearchStage.offlineFallback: + _renderOfflineOutcome( + event.offline!, + OfflineUxState.lowBandwidthLocalFallback, + event.generation, + ); + _recordSearchTelemetry( + traceId: traceId, + dispatchedAt: dispatchedAt, + connectivity: connectivity, + source: SearchResultSource.offlineFallback, + localParserCouldAnswer: true, + fallbackTriggerMs: event.elapsedMs, + timeToFirstResultMs: event.elapsedMs, + ); + + case SearchStage.offlineAfterOnlineFailure: + _renderOfflineOutcome( + event.offline!, + OfflineUxState.backendUnreachableLocalAvailable, + event.generation, + ); + _recordSearchTelemetry( + traceId: traceId, + dispatchedAt: dispatchedAt, + connectivity: connectivity, + source: SearchResultSource.offlineFallback, + localParserCouldAnswer: true, + fallbackTriggerMs: event.elapsedMs, + timeToFirstResultMs: event.elapsedMs, + ); + + case SearchStage.lateOnlineAvailable: + final turn = event.online!; + if (!_isOfferableOnlineTurn(turn)) { + // The request completed, but with an error rather than an answer. + // Offering "fresh results" that are actually a failure would be a + // worse trade than the saved data already on screen, so this is + // treated exactly like a late failure. + setState(() { + _pendingOnlineTurn = null; + _offlineState = OfflineUxState.backendUnreachableLocalAvailable; + }); + return; + } + // The authoritative answer is ready, but saved data is on screen. + // Offer it; never swap it in. setState(() { - if (backendSession != null) _session = backendSession; - _isLoading = false; - _clarificationQuestion = result.clarificationQuestion; - _clarificationCandidates = result.candidates; - final pendingQuery = result.parsedQuery ?? result.query; - _pendingClarification = pendingQuery == null - ? null - : PendingClarification( - query: pendingQuery, - referents: result.referents, - requestId: nextRequestId, - ); + _pendingOnlineTurn = turn; + _pendingOnlineQueryText = queryText; + _pendingOnlineGeneration = event.generation; }); - return; - } - if (!result.isSuccess || result.query == null) { + case SearchStage.lateOnlineFailed: + // Local data stays exactly as it is; only the label changes. setState(() { - if (backendSession != null) _session = backendSession; - _isLoading = false; - _errorKind = _SearchErrorKind.understanding; - _errorMessage = result.friendlyMessage ?? 'Query parsing failed'; - _clarificationCandidates = []; + _pendingOnlineTurn = null; + _offlineState = OfflineUxState.backendUnreachableLocalAvailable; }); - return; - } - final parsedQuery = result.query!; - final l10n = AppLocalizations.of(context); - final localizedSummary = _buildLocalizedInterpretedSummary( - parsedQuery, - l10n, - ); + case SearchStage.onlineFailed: + await _renderSearchFailure(event.error, connectivity, event.generation); + _recordSearchTelemetry( + traceId: traceId, + dispatchedAt: dispatchedAt, + connectivity: connectivity, + source: SearchResultSource.online, + ); + } + } - // Determine which fields were inherited vs refined in this turn - final inherited = {}; - final refinements = {}; + /// Promotes the offered authoritative result. Only ever reached by an + /// explicit user tap on "show latest". + Future _showLatestOnlineResult() async { + final turn = _pendingOnlineTurn; + final queryText = _pendingOnlineQueryText; + final generation = _pendingOnlineGeneration; + // Clearing first makes repeated taps idempotent: the second tap finds no + // pending turn and does nothing. + _discardPendingOnline(); + if (turn == null || queryText == null || generation == null) return; + if (!mounted || _session.activeRequestId != generation) return; + setState(() { + _offlineState = OfflineUxState.online; + _offlineAge = null; + }); + await _applyOnlineTurn(turn, queryText, generation); + } - if (parsedQuery.rallyNames.isNotEmpty) { - if (_session.activeQuery.rallyNames.isNotEmpty && - _session.activeQuery.rallyNames.first == - parsedQuery.rallyNames.first) { - inherited.add('rally'); - } else { - refinements.add('rally'); - } - } - if (parsedQuery.driverNames.isNotEmpty) { - if (_session.activeQuery.driverNames.isNotEmpty && - _session.activeQuery.driverNames.first == - parsedQuery.driverNames.first) { - inherited.add('driver'); - } else { - refinements.add('driver'); - } - } - if (parsedQuery.actionTypes.isNotEmpty) { - refinements.add('action'); - } - if (parsedQuery.countries.isNotEmpty) { - if (_session.activeQuery.countries.isNotEmpty && - _session.activeQuery.countries.first == - parsedQuery.countries.first) { - inherited.add('country'); - } else { - refinements.add('country'); - } + /// Whether a late online turn is worth offering in place of saved data. + /// + /// A clarification or a special response is a real authoritative outcome and + /// is offered; a parse or backend error is not. + static bool _isOfferableOnlineTurn(_OnlineTurn turn) { + final result = turn.result; + if (result.isSpecialResponse || result.requiresClarification) return true; + return result.isSuccess && result.query != null; + } + + void _discardPendingOnline() { + _pendingOnlineTurn = null; + _pendingOnlineQueryText = null; + _pendingOnlineGeneration = null; + } + + void _recordSearchTelemetry({ + required String traceId, + required DateTime dispatchedAt, + required ConnectivityState connectivity, + required SearchResultSource source, + int? networkRoundtripMs, + int? timeToFirstResultMs, + int? fallbackTriggerMs, + bool localParserCouldAnswer = false, + }) { + _telemetrySink.record( + SearchTelemetry( + requestId: traceId, + totalClientMs: clock.now().difference(dispatchedAt).inMilliseconds, + resultSource: source, + connectivity: connectivity, + fallbackTriggered: fallbackTriggerMs != null, + fallbackTriggerMs: fallbackTriggerMs, + networkRoundtripMs: networkRoundtripMs, + timeToFirstResultMs: timeToFirstResultMs, + localParserCouldAnswer: localParserCouldAnswer, + lateOnlineOffered: _pendingOnlineTurn != null, + ), + ); + } + + Future _renderSearchFailure( + Object? error, + ConnectivityState connectivity, + int generation, + ) async { + // A plainly-offline device with no snapshot is a different product state + // from a backend failure, and gets its own "sync now" affordance. + if (connectivity == ConnectivityState.offline && _offlineEngine != null) { + var hasSnapshot = false; + try { + hasSnapshot = await _offlineEngine!.database.hasSnapshot(); + } catch (_) { + hasSnapshot = false; } - if (parsedQuery.years.isNotEmpty) { - if (_session.activeQuery.years.isNotEmpty && - _session.activeQuery.years.first == parsedQuery.years.first) { - inherited.add('year'); - } else { - refinements.add('year'); - } + if (!mounted || _session.activeRequestId != generation) return; + if (!hasSnapshot) { + setState(() { + _isLoading = false; + _searchResponse = null; + _totalCount = 0; + _errorMessage = null; + _offlineState = OfflineUxState.noLocalSnapshot; + _offlineAge = null; + }); + return; } + } + if (!mounted || _session.activeRequestId != generation) return; + setState(() { + _errorKind = _SearchErrorKind.service; + _errorMessage = error is PythonApiException + ? error.friendlyMessage + : const FriendlyResponseService().responseFor( + FriendlyResponseCategory.serverError, + ); + _clarificationCandidates = []; + _isLoading = false; + }); + } - final updatedSession = - backendSession ?? - _session.recordTurn( - query: parsedQuery, - referents: result.referents, - title: queryText, - response: result.searchResponse, - interpretedSummary: localizedSummary, - inherited: inherited, - refinements: refinements, - ); + /// Renders one authoritative online turn onto shared result state. + /// + /// Shared by the in-budget path and by "show latest", so a promoted late + /// result is applied through exactly the same code as a fast one. + Future _applyOnlineTurn( + _OnlineTurn turn, + String queryText, + int generation, + ) async { + final result = turn.result; + final backendSession = turn.session; + if (turn.echoedRequestId != null && turn.echoedRequestId != generation) { + return; + } + if (!mounted || _session.activeRequestId != generation) return; + if (turn.transcript != null && turn.transcript!.isNotEmpty) { + _searchController.text = turn.transcript!; + } + if (result.isSpecialResponse) { + // Easter-egg / personality response: show only the playful message. + // No DB results, no zero-results state, no clarification, no stale + // pagination. The rally conversation context (referents/active query) + // from any prior search is intentionally preserved. setState(() { - _session = updatedSession; - _searchResponse = result.searchResponse; - _totalCount = result.totalCount; - _lastNlResult = result; + if (backendSession != null) _session = backendSession; + _isLoading = false; + _specialMessage = result.friendlyMessage; + _searchResponse = null; + _totalCount = 0; _clarificationQuestion = null; _clarificationCandidates = []; _pendingClarification = null; _errorMessage = null; - _specialMessage = null; - _emptyResultsMessage = result.friendlyMessage; - _isLoading = false; - // Authoritative online answer: clear any offline chrome. + _emptyResultsMessage = null; _offlineState = OfflineUxState.online; _offlineAge = null; }); - // Opportunistic, non-blocking refresh of the offline snapshot. - unawaited(_offlineSync?.maybeSync() ?? Future.value()); - } catch (e) { - if (!mounted || _session.activeRequestId != nextRequestId) return; - // The authoritative backend failed. If we have a local snapshot, answer - // from it (clearly labelled) rather than showing a dead end. - if (queryText.isNotEmpty && _offlineEngine != null && e is PythonApiException) { - await _runOfflineFallback(queryText, OfflineUxState.backendUnreachableLocalAvailable); - if (_usePythonBackend && spokenResult?.audioContext != null) { - spokenResult!.disposeAudio(); - } - return; - } + return; + } + + if (result.requiresClarification) { setState(() { - _errorKind = _SearchErrorKind.service; - _errorMessage = e is PythonApiException - ? e.friendlyMessage - : const FriendlyResponseService().responseFor( - FriendlyResponseCategory.serverError, + if (backendSession != null) _session = backendSession; + _isLoading = false; + _clarificationQuestion = result.clarificationQuestion; + _clarificationCandidates = result.candidates; + final pendingQuery = result.parsedQuery ?? result.query; + _pendingClarification = pendingQuery == null + ? null + : PendingClarification( + query: pendingQuery, + referents: result.referents, + requestId: generation, ); - _clarificationCandidates = []; + _offlineState = OfflineUxState.online; + _offlineAge = null; + }); + return; + } + + if (!result.isSuccess || result.query == null) { + setState(() { + if (backendSession != null) _session = backendSession; _isLoading = false; + _errorKind = _SearchErrorKind.understanding; + _errorMessage = result.friendlyMessage ?? 'Query parsing failed'; + _clarificationCandidates = []; + _offlineState = OfflineUxState.online; + _offlineAge = null; }); - } finally { - if (_usePythonBackend && spokenResult?.audioContext != null) { - spokenResult!.disposeAudio(); + return; + } + + final parsedQuery = result.query!; + final l10n = AppLocalizations.of(context); + final localizedSummary = _buildLocalizedInterpretedSummary( + parsedQuery, + l10n, + ); + + // Determine which fields were inherited vs refined in this turn + final inherited = {}; + final refinements = {}; + + if (parsedQuery.rallyNames.isNotEmpty) { + if (_session.activeQuery.rallyNames.isNotEmpty && + _session.activeQuery.rallyNames.first == + parsedQuery.rallyNames.first) { + inherited.add('rally'); + } else { + refinements.add('rally'); } } - } + if (parsedQuery.driverNames.isNotEmpty) { + if (_session.activeQuery.driverNames.isNotEmpty && + _session.activeQuery.driverNames.first == + parsedQuery.driverNames.first) { + inherited.add('driver'); + } else { + refinements.add('driver'); + } + } + if (parsedQuery.actionTypes.isNotEmpty) { + refinements.add('action'); + } + if (parsedQuery.countries.isNotEmpty) { + if (_session.activeQuery.countries.isNotEmpty && + _session.activeQuery.countries.first == parsedQuery.countries.first) { + inherited.add('country'); + } else { + refinements.add('country'); + } + } + if (parsedQuery.years.isNotEmpty) { + if (_session.activeQuery.years.isNotEmpty && + _session.activeQuery.years.first == parsedQuery.years.first) { + inherited.add('year'); + } else { + refinements.add('year'); + } + } + + final updatedSession = + backendSession ?? + _session.recordTurn( + query: parsedQuery, + referents: result.referents, + title: queryText, + response: result.searchResponse, + interpretedSummary: localizedSummary, + inherited: inherited, + refinements: refinements, + ); + setState(() { + _session = updatedSession; + _searchResponse = result.searchResponse; + _totalCount = result.totalCount; + _lastNlResult = result; + _clarificationQuestion = null; + _clarificationCandidates = []; + _pendingClarification = null; + _errorMessage = null; + _specialMessage = null; + _emptyResultsMessage = result.friendlyMessage; + _isLoading = false; + // Authoritative online answer: clear any offline chrome. + _offlineState = OfflineUxState.online; + _offlineAge = null; + }); + // Opportunistic, non-blocking refresh of the offline snapshot. + unawaited(_offlineSync?.maybeSync() ?? Future.value()); + } Future _executeDeterministicSearch({bool resetPage = false}) async { if (resetPage) { _currentPage = 1; @@ -608,15 +911,11 @@ class _GeneralSearchScreenState extends State { // OFFLINE FALLBACK // =========================================================================== - Future _isPlainlyOffline() async { - final probe = _connectivityProbe; - if (probe == null) return false; - try { - return !(await probe.isOnline()); - } catch (_) { - return false; - } - } + /// Connectivity for the deterministic (pagination / clarification) path. + /// Reads through the coordinator so the whole screen has one notion of + /// "known offline" rather than two probes that can disagree. + Future _isPlainlyOffline() async => + await _coordinator.observedConnectivity() == ConnectivityState.offline; Future _snapshotAge() async { final last = await _offlineEngine?.database.lastSyncUtc(); @@ -624,47 +923,23 @@ class _GeneralSearchScreenState extends State { return DateTime.now().toUtc().difference(last); } - static const Duration _staleThreshold = Duration(hours: 12); - - /// Runs the deterministic offline pipeline for a raw query and maps its - /// outcome onto the shared result/clarification/special/error state, plus the - /// product-tone offline banner state. - Future _runOfflineFallback(String queryText, OfflineUxState baseState) async { - final engine = _offlineEngine; - if (engine == null) return; - if (!await engine.database.hasSnapshot()) { - if (!mounted) return; - setState(() { - _isLoading = false; - _searchResponse = null; - _totalCount = 0; - _clarificationQuestion = null; - _clarificationCandidates = []; - _pendingClarification = null; - _specialMessage = null; - _emptyResultsMessage = null; - _errorMessage = null; - _offlineState = OfflineUxState.noLocalSnapshot; - _offlineAge = null; - }); - return; - } + /// A snapshot older than this is still used, but is labelled with its age. + /// The value comes from the single latency policy. + Duration get _staleThreshold => widget.latencyPolicy.snapshotStaleAfter; + + /// Maps an offline outcome the coordinator already computed onto the shared + /// result / clarification / special / error state, plus the product-tone + /// offline banner state. + /// + /// Render-only: the offline pipeline is executed once, by the coordinator, + /// so a fallback never re-runs the local search just to display it. + Future _renderOfflineOutcome( + OfflineSearchOutcome outcome, + OfflineUxState baseState, + int generation, + ) async { final age = await _snapshotAge(); - OfflineSearchOutcome outcome; - try { - outcome = await engine.search(queryText, limit: _pageSize, offset: (_currentPage - 1) * _pageSize); - } catch (_) { - if (!mounted) return; - setState(() { - _isLoading = false; - _searchResponse = null; - _totalCount = 0; - _offlineState = OfflineUxState.offlineSafeNoMatch; - _emptyResultsMessage = const FriendlyResponseService().responseFor(FriendlyResponseCategory.noResults); - }); - return; - } - if (!mounted) return; + if (!mounted || _session.activeRequestId != generation) return; setState(() { _isLoading = false; _hasSearched = true; @@ -1080,19 +1355,6 @@ class _GeneralSearchScreenState extends State { onPressed: _openAdvancedFilters, ), - // Browse the raw stream registry (secondary area). - IconButton( - tooltip: 'Browse streams', - icon: const Icon(Icons.video_library_outlined), - onPressed: () { - Navigator.of(context).push( - MaterialPageRoute( - builder: (context) => const RallyStreamsPage(), - ), - ); - }, - ), - // Reset Session Button IconButton( tooltip: 'Reset Session', @@ -1306,6 +1568,18 @@ class _GeneralSearchScreenState extends State { : null, ), + // 2c. A late authoritative result, offered rather than applied. The + // saved data stays on screen until the user taps through, so the + // result under their eyes never changes on its own. + if (_hasPendingOnlineResult) + OfflineBanner( + key: const Key('freshResultsBanner'), + state: OfflineUxState.lateOnlineResultAvailable, + actionLabel: 'Show latest', + onAction: () => unawaited(_showLatestOnlineResult()), + onDismiss: () => setState(_discardPendingOnline), + ), + // 3. Inline Clarification Card (if disambiguation required) if (_clarificationQuestion != null) ClarificationCard( diff --git a/lib/screens/rally_streams_page.dart b/lib/screens/rally_streams_page.dart deleted file mode 100644 index 4708fb0..0000000 --- a/lib/screens/rally_streams_page.dart +++ /dev/null @@ -1,1364 +0,0 @@ -import 'package:flutter/material.dart'; -import 'package:flutter/services.dart'; -import '../models/rally_stream.dart'; -import '../models/video_action.dart'; -import '../services/database_service.dart'; -import '../services/video_action_repository.dart'; -import '../widgets/rally_video_player.dart'; -import '../widgets/video_action_card.dart'; -import '../widgets/action_player_modal.dart'; -import 'general_search_screen.dart'; -import 'video_action_search_screen.dart'; - -class RallyStreamsPage extends StatefulWidget { - const RallyStreamsPage({super.key}); - - @override - State createState() => _RallyStreamsPageState(); -} - -class _RallyStreamsPageState extends State { - final DatabaseService _dbService = DatabaseService(); - final VideoActionRepository _actionRepo = VideoActionRepository(); - - List _streams = []; - bool _isLoading = false; - String? _errorMessage; - - // Pagination state - Default limit is 10 - int _currentPage = 1; - int _pageSize = 10; - int _totalCount = 0; - - // Search & Filter state - final TextEditingController _searchController = TextEditingController(); - String _searchQuery = ''; - String _selectedVideoType = 'ALL'; - String _selectedClipStatus = 'ALL'; - String _sortBy = 'id'; - bool _sortAscending = false; - - final List _videoTypes = [ - 'ALL', - 'sendObs', - 'startFirstVideo', - 'nonLive', - ]; - - final List _clipStatuses = [ - 'ALL', - 'complete', - 'live', - ]; - - @override - void initState() { - super.initState(); - _fetchStreams(); - } - - @override - void dispose() { - _searchController.dispose(); - super.dispose(); - } - - Future _fetchStreams({bool resetPage = false}) async { - if (resetPage) { - _currentPage = 1; - } - - setState(() { - _isLoading = true; - _errorMessage = null; - }); - - try { - final offset = (_currentPage - 1) * _pageSize; - - final total = await _dbService.getRallyStreamsCount( - searchQuery: _searchQuery, - videoType: _selectedVideoType, - clipStatus: _selectedClipStatus, - ); - - final rows = await _dbService.getRallyStreams( - limit: _pageSize, - offset: offset, - searchQuery: _searchQuery, - videoType: _selectedVideoType, - clipStatus: _selectedClipStatus, - sortBy: _sortBy, - sortAscending: _sortAscending, - ); - - final streams = rows.map((r) => RallyStream.fromMap(r)).toList(); - - if (mounted) { - setState(() { - _streams = streams; - _totalCount = total; - _isLoading = false; - }); - } - } catch (e) { - if (mounted) { - setState(() { - _errorMessage = e.toString(); - _isLoading = false; - }); - } - } - } - - int get _totalPages => (_totalCount / _pageSize).ceil().clamp(1, 999999); - - void _goToPage(int page) { - if (page < 1 || page > _totalPages || page == _currentPage) return; - setState(() { - _currentPage = page; - }); - _fetchStreams(); - } - - void _showJumpToPageDialog() { - final textController = TextEditingController(text: _currentPage.toString()); - showDialog( - context: context, - builder: (ctx) { - return AlertDialog( - title: const Text('Jump to Page'), - content: TextField( - controller: textController, - keyboardType: TextInputType.number, - autofocus: true, - decoration: InputDecoration( - hintText: 'Enter page (1 - $_totalPages)', - border: const OutlineInputBorder(), - ), - ), - actions: [ - TextButton( - onPressed: () => Navigator.pop(ctx), - child: const Text('Cancel'), - ), - FilledButton( - onPressed: () { - final page = int.tryParse(textController.text.trim()); - if (page != null && page >= 1 && page <= _totalPages) { - Navigator.pop(ctx); - _goToPage(page); - } - }, - child: const Text('Go'), - ), - ], - ); - }, - ); - } - - @override - Widget build(BuildContext context) { - final theme = Theme.of(context); - final isDark = theme.brightness == Brightness.dark; - - final screenWidth = MediaQuery.sizeOf(context).width; - final isCompact = screenWidth < 768; - - return Scaffold( - backgroundColor: isDark ? const Color(0xFF121212) : const Color(0xFFF7F9FC), - appBar: AppBar( - elevation: 0, - backgroundColor: isDark ? const Color(0xFF1E1E1E) : Colors.white, - title: LayoutBuilder( - builder: (context, constraints) { - if (constraints.maxWidth < 40) { - return Icon( - Icons.play_circle_filled_rounded, - color: theme.colorScheme.primary, - size: 20, - ); - } - return FittedBox( - fit: BoxFit.scaleDown, - alignment: Alignment.centerLeft, - child: Row( - mainAxisSize: MainAxisSize.min, - children: [ - Container( - padding: const EdgeInsets.all(7), - decoration: BoxDecoration( - color: theme.colorScheme.primaryContainer, - borderRadius: BorderRadius.circular(8), - ), - child: Icon( - Icons.play_circle_filled_rounded, - color: theme.colorScheme.primary, - size: 20, - ), - ), - const SizedBox(width: 10), - Column( - crossAxisAlignment: CrossAxisAlignment.start, - mainAxisSize: MainAxisSize.min, - children: [ - const Text( - 'Rally Streams', - style: TextStyle(fontSize: 17, fontWeight: FontWeight.bold), - ), - if (constraints.maxWidth > 180) - Text( - _totalCount > 0 - ? '$_totalCount playable streams in database' - : 'Database Video Player Registry', - style: theme.textTheme.bodySmall?.copyWith( - color: theme.textTheme.bodySmall?.color?.withValues(alpha: 0.7), - fontSize: 11, - ), - ), - ], - ), - ], - ), - ); - }, - ), - actions: [ - if (isCompact) ...[ - IconButton( - tooltip: 'General Search', - icon: const Icon(Icons.search_rounded), - onPressed: () { - Navigator.of(context).push( - MaterialPageRoute( - builder: (context) => const GeneralSearchScreen(), - ), - ); - }, - ), - IconButton( - tooltip: 'Moments Search', - icon: const Icon(Icons.bolt_rounded), - onPressed: () { - Navigator.of(context).push( - MaterialPageRoute( - builder: (context) => const VideoActionSearchScreen(), - ), - ); - }, - ), - ] else ...[ - FilledButton.icon( - onPressed: () { - Navigator.of(context).push( - MaterialPageRoute( - builder: (context) => const GeneralSearchScreen(), - ), - ); - }, - icon: const Icon(Icons.search_rounded, size: 18), - label: const Text('General Search'), - style: FilledButton.styleFrom( - padding: const EdgeInsets.symmetric(horizontal: 12, vertical: 6), - ), - ), - const SizedBox(width: 8), - FilledButton.tonalIcon( - onPressed: () { - Navigator.of(context).push( - MaterialPageRoute( - builder: (context) => const VideoActionSearchScreen(), - ), - ); - }, - icon: const Icon(Icons.bolt_rounded, size: 18), - label: const Text('Moments'), - style: FilledButton.styleFrom( - padding: const EdgeInsets.symmetric(horizontal: 12, vertical: 6), - ), - ), - ], - const SizedBox(width: 4), - IconButton( - tooltip: 'Refresh', - icon: _isLoading - ? const SizedBox( - width: 20, - height: 20, - child: CircularProgressIndicator(strokeWidth: 2), - ) - : const Icon(Icons.refresh_rounded), - onPressed: _isLoading ? null : () => _fetchStreams(), - ), - const SizedBox(width: 8), - ], - ), - body: Column( - children: [ - // Filter & Search Header - _buildFilterBar(context), - - // Main Content - Expanded( - child: _buildBody(context), - ), - - // Pagination Bottom Bar - _buildPaginationBar(context), - ], - ), - ); - } - - Widget _buildFilterBar(BuildContext context) { - final theme = Theme.of(context); - final isDark = theme.brightness == Brightness.dark; - - return Container( - padding: const EdgeInsets.symmetric(horizontal: 16, vertical: 12), - decoration: BoxDecoration( - color: isDark ? const Color(0xFF1A1A1A) : Colors.white, - border: Border( - bottom: BorderSide( - color: isDark ? Colors.white10 : Colors.black.withValues(alpha: 0.06), - ), - ), - ), - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - // Search Box & Action Row - Row( - children: [ - Expanded( - child: TextField( - controller: _searchController, - decoration: InputDecoration( - hintText: 'Search by Stream ID, Video ID, or URL...', - hintStyle: TextStyle(fontSize: 13, color: theme.hintColor), - prefixIcon: const Icon(Icons.search, size: 20), - suffixIcon: _searchQuery.isNotEmpty - ? IconButton( - icon: const Icon(Icons.clear, size: 18), - onPressed: () { - _searchController.clear(); - setState(() { - _searchQuery = ''; - }); - _fetchStreams(resetPage: true); - }, - ) - : null, - filled: true, - fillColor: isDark ? const Color(0xFF252525) : const Color(0xFFF1F4F9), - border: OutlineInputBorder( - borderRadius: BorderRadius.circular(10), - borderSide: BorderSide.none, - ), - contentPadding: const EdgeInsets.symmetric( - horizontal: 14, - vertical: 10, - ), - isDense: true, - ), - onSubmitted: (val) { - setState(() { - _searchQuery = val; - }); - _fetchStreams(resetPage: true); - }, - ), - ), - const SizedBox(width: 8), - FilledButton.tonalIcon( - onPressed: () { - setState(() { - _searchQuery = _searchController.text; - }); - _fetchStreams(resetPage: true); - }, - icon: const Icon(Icons.filter_list_rounded, size: 18), - label: const Text('Filter'), - style: FilledButton.styleFrom( - padding: const EdgeInsets.symmetric(horizontal: 14, vertical: 12), - shape: RoundedRectangleBorder( - borderRadius: BorderRadius.circular(10), - ), - ), - ), - ], - ), - const SizedBox(height: 10), - - // Filters Row - SingleChildScrollView( - scrollDirection: Axis.horizontal, - child: Row( - children: [ - // Video Type Dropdown - _buildFilterChipDropdown( - label: 'Type: ${_selectedVideoType == 'ALL' ? 'All Types' : _selectedVideoType}', - icon: Icons.category_rounded, - items: _videoTypes, - selectedItem: _selectedVideoType, - onChanged: (val) { - if (val != null && val != _selectedVideoType) { - setState(() { - _selectedVideoType = val; - }); - _fetchStreams(resetPage: true); - } - }, - ), - const SizedBox(width: 8), - - // Status Dropdown - _buildFilterChipDropdown( - label: 'Status: ${_selectedClipStatus == 'ALL' ? 'All Statuses' : _selectedClipStatus}', - icon: Icons.flag_rounded, - items: _clipStatuses, - selectedItem: _selectedClipStatus, - onChanged: (val) { - if (val != null && val != _selectedClipStatus) { - setState(() { - _selectedClipStatus = val; - }); - _fetchStreams(resetPage: true); - } - }, - ), - const SizedBox(width: 8), - - // Sort Dropdown - _buildSortDropdown(context), - - if (_searchQuery.isNotEmpty || - _selectedVideoType != 'ALL' || - _selectedClipStatus != 'ALL' || - _sortBy != 'id' || - _sortAscending != false) ...[ - const SizedBox(width: 8), - TextButton.icon( - onPressed: () { - _searchController.clear(); - setState(() { - _searchQuery = ''; - _selectedVideoType = 'ALL'; - _selectedClipStatus = 'ALL'; - _sortBy = 'id'; - _sortAscending = false; - }); - _fetchStreams(resetPage: true); - }, - icon: const Icon(Icons.restart_alt, size: 16), - label: const Text('Reset', style: TextStyle(fontSize: 12)), - style: TextButton.styleFrom( - padding: const EdgeInsets.symmetric(horizontal: 10, vertical: 6), - ), - ), - ], - ], - ), - ), - ], - ), - ); - } - - Widget _buildFilterChipDropdown({ - required String label, - required IconData icon, - required List items, - required String selectedItem, - required ValueChanged onChanged, - }) { - final theme = Theme.of(context); - final isDark = theme.brightness == Brightness.dark; - - return Container( - padding: const EdgeInsets.symmetric(horizontal: 10, vertical: 4), - decoration: BoxDecoration( - color: isDark ? const Color(0xFF252525) : const Color(0xFFF1F4F9), - borderRadius: BorderRadius.circular(8), - border: Border.all( - color: selectedItem != 'ALL' - ? theme.colorScheme.primary.withValues(alpha: 0.5) - : Colors.transparent, - ), - ), - child: DropdownButtonHideUnderline( - child: DropdownButton( - value: selectedItem, - isDense: true, - icon: const Icon(Icons.arrow_drop_down, size: 18), - items: items.map((type) { - return DropdownMenuItem( - value: type, - child: Row( - mainAxisSize: MainAxisSize.min, - children: [ - Icon( - icon, - size: 14, - color: selectedItem == type ? theme.colorScheme.primary : theme.hintColor, - ), - const SizedBox(width: 6), - Text( - type == 'ALL' ? 'All' : type, - style: TextStyle( - fontSize: 12, - fontWeight: selectedItem == type ? FontWeight.bold : FontWeight.normal, - ), - ), - ], - ), - ); - }).toList(), - onChanged: onChanged, - ), - ), - ); - } - - Widget _buildSortDropdown(BuildContext context) { - final theme = Theme.of(context); - final isDark = theme.brightness == Brightness.dark; - - return Container( - padding: const EdgeInsets.symmetric(horizontal: 10, vertical: 4), - decoration: BoxDecoration( - color: isDark ? const Color(0xFF252525) : const Color(0xFFF1F4F9), - borderRadius: BorderRadius.circular(8), - ), - child: DropdownButtonHideUnderline( - child: DropdownButton( - value: '$_sortBy:$_sortAscending', - isDense: true, - icon: const Icon(Icons.arrow_drop_down, size: 18), - items: const [ - DropdownMenuItem( - value: 'id:false', - child: Text('Newest ID First', style: TextStyle(fontSize: 12)), - ), - DropdownMenuItem( - value: 'id:true', - child: Text('Oldest ID First', style: TextStyle(fontSize: 12)), - ), - DropdownMenuItem( - value: 'clip_duration:false', - child: Text('Longest Duration', style: TextStyle(fontSize: 12)), - ), - DropdownMenuItem( - value: 'clip_duration:true', - child: Text('Shortest Duration', style: TextStyle(fontSize: 12)), - ), - DropdownMenuItem( - value: 'share_counter:false', - child: Text('Most Shares', style: TextStyle(fontSize: 12)), - ), - DropdownMenuItem( - value: 'download_counter:false', - child: Text('Most Downloads', style: TextStyle(fontSize: 12)), - ), - ], - onChanged: (val) { - if (val != null) { - final parts = val.split(':'); - setState(() { - _sortBy = parts[0]; - _sortAscending = parts[1] == 'true'; - }); - _fetchStreams(resetPage: true); - } - }, - ), - ), - ); - } - - Widget _buildBody(BuildContext context) { - if (_isLoading && _streams.isEmpty) { - return const Center( - child: Column( - mainAxisAlignment: MainAxisAlignment.center, - children: [ - CircularProgressIndicator(), - SizedBox(height: 16), - Text('Loading playable streams from database...'), - ], - ), - ); - } - - if (_errorMessage != null && _streams.isEmpty) { - return Center( - child: Padding( - padding: const EdgeInsets.all(24.0), - child: Column( - mainAxisAlignment: MainAxisAlignment.center, - children: [ - const Icon(Icons.error_outline_rounded, color: Colors.red, size: 48), - const SizedBox(height: 16), - const Text( - 'Failed to load rally streams', - style: TextStyle(fontSize: 18, fontWeight: FontWeight.bold), - ), - const SizedBox(height: 8), - Text( - _errorMessage!, - textAlign: TextAlign.center, - style: TextStyle(color: Theme.of(context).hintColor, fontSize: 13), - ), - const SizedBox(height: 20), - FilledButton.icon( - onPressed: () => _fetchStreams(), - icon: const Icon(Icons.refresh), - label: const Text('Try Again'), - ), - ], - ), - ), - ); - } - - if (_streams.isEmpty) { - return Center( - child: Column( - mainAxisAlignment: MainAxisAlignment.center, - children: [ - Icon(Icons.search_off_rounded, size: 56, color: Theme.of(context).hintColor), - const SizedBox(height: 16), - const Text( - 'No streams found', - style: TextStyle(fontSize: 18, fontWeight: FontWeight.bold), - ), - const SizedBox(height: 6), - Text( - 'Try adjusting your search query or filters', - style: TextStyle(color: Theme.of(context).hintColor), - ), - ], - ), - ); - } - - return RefreshIndicator( - onRefresh: () => _fetchStreams(), - child: Stack( - children: [ - ListView.builder( - padding: const EdgeInsets.all(16), - itemCount: _streams.length, - itemBuilder: (context, index) { - final stream = _streams[index]; - return _buildStreamCard(context, stream); - }, - ), - if (_isLoading) - Positioned( - top: 0, - left: 0, - right: 0, - child: LinearProgressIndicator( - backgroundColor: Colors.transparent, - valueColor: AlwaysStoppedAnimation( - Theme.of(context).colorScheme.primary, - ), - ), - ), - ], - ), - ); - } - - Widget _buildStreamCard(BuildContext context, RallyStream stream) { - final theme = Theme.of(context); - final isDark = theme.brightness == Brightness.dark; - - final typeColor = _getVideoTypeColor(stream.videoType); - final statusColor = _getStatusColor(stream.clipStatus); - - return Card( - margin: const EdgeInsets.only(bottom: 18), - elevation: 0, - shape: RoundedRectangleBorder( - borderRadius: BorderRadius.circular(16), - side: BorderSide( - color: isDark ? Colors.white10 : Colors.black.withValues(alpha: 0.08), - ), - ), - color: isDark ? const Color(0xFF1E1E1E) : Colors.white, - child: Padding( - padding: const EdgeInsets.all(16.0), - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - // Top Row: Stream ID, Video ID, Video Type, and Status - Wrap( - alignment: WrapAlignment.spaceBetween, - crossAxisAlignment: WrapCrossAlignment.center, - spacing: 8, - runSpacing: 8, - children: [ - // Left group: Stream ID & Video ID - Row( - mainAxisSize: MainAxisSize.min, - children: [ - // Stream ID Badge - Container( - padding: const EdgeInsets.symmetric(horizontal: 10, vertical: 4), - decoration: BoxDecoration( - color: theme.colorScheme.primary.withValues(alpha: 0.12), - borderRadius: BorderRadius.circular(8), - ), - child: Row( - mainAxisSize: MainAxisSize.min, - children: [ - Icon(Icons.tag_rounded, size: 14, color: theme.colorScheme.primary), - Text( - '${stream.id}', - style: TextStyle( - fontWeight: FontWeight.bold, - fontSize: 13, - color: theme.colorScheme.primary, - fontFamily: 'monospace', - ), - ), - ], - ), - ), - const SizedBox(width: 8), - - // Video ID Badge - if (stream.videoId != null) ...[ - Container( - padding: const EdgeInsets.symmetric(horizontal: 8, vertical: 4), - decoration: BoxDecoration( - color: isDark ? Colors.white12 : Colors.black.withValues(alpha: 0.06), - borderRadius: BorderRadius.circular(8), - ), - child: Text( - 'Video #${stream.videoId}', - style: const TextStyle( - fontSize: 12, - fontWeight: FontWeight.w600, - fontFamily: 'monospace', - ), - ), - ), - ], - ], - ), - - // Right group: Video Type & Status Chips - Row( - mainAxisSize: MainAxisSize.min, - children: [ - // Video Type Chip - if (stream.videoType != null) ...[ - Container( - padding: const EdgeInsets.symmetric(horizontal: 8, vertical: 4), - decoration: BoxDecoration( - color: typeColor.withValues(alpha: 0.15), - borderRadius: BorderRadius.circular(8), - border: Border.all(color: typeColor.withValues(alpha: 0.4)), - ), - child: Text( - stream.videoType!, - style: TextStyle( - fontSize: 11, - fontWeight: FontWeight.bold, - color: typeColor, - ), - ), - ), - const SizedBox(width: 6), - ], - - // Clip Status Chip - if (stream.clipStatus != null) ...[ - Container( - padding: const EdgeInsets.symmetric(horizontal: 8, vertical: 4), - decoration: BoxDecoration( - color: statusColor.withValues(alpha: 0.15), - borderRadius: BorderRadius.circular(8), - ), - child: Row( - mainAxisSize: MainAxisSize.min, - children: [ - Container( - width: 6, - height: 6, - decoration: BoxDecoration( - shape: BoxShape.circle, - color: statusColor, - ), - ), - const SizedBox(width: 5), - Text( - stream.clipStatus!, - style: TextStyle( - fontSize: 11, - fontWeight: FontWeight.w600, - color: statusColor, - ), - ), - ], - ), - ), - ], - ], - ), - ], - ), - const SizedBox(height: 12), - - // Embedded Playable Video Player - if (stream.onDemandUrl != null && stream.onDemandUrl!.isNotEmpty) ...[ - RallyVideoPlayer( - key: ValueKey('player_${stream.id}_${stream.onDemandUrl}_${stream.clipStartTime}_${stream.clipDuration}'), - videoUrl: stream.onDemandUrl!, - videoTitle: 'Stream #${stream.id} • Video #${stream.videoId ?? 'N/A'}', - initialStartTime: stream.clipStartTime, - initialEndTime: (stream.clipDuration != null && stream.clipDuration! > 0) - ? ((stream.clipStartTime ?? 0.0) + stream.clipDuration!) - : null, - autoPlay: false, - ), - const SizedBox(height: 12), - ] else ...[ - Container( - height: 120, - alignment: Alignment.center, - decoration: BoxDecoration( - color: isDark ? Colors.black26 : const Color(0xFFF0F2F5), - borderRadius: BorderRadius.circular(12), - ), - child: Row( - mainAxisAlignment: MainAxisAlignment.center, - children: [ - Icon(Icons.videocam_off_rounded, color: theme.hintColor), - const SizedBox(width: 8), - Text( - 'No video stream available for this entry', - style: TextStyle(color: theme.hintColor, fontSize: 13), - ), - ], - ), - ), - const SizedBox(height: 12), - ], - - // Metadata Info: Duration, Clip Window, Start Time, Created At, Counters - Wrap( - spacing: 16, - runSpacing: 8, - children: [ - _buildMetaInfo( - icon: Icons.timer_outlined, - label: 'Clip Duration', - value: stream.formattedDuration, - ), - _buildMetaInfo( - icon: Icons.timelapse_rounded, - label: 'Clip Window', - value: stream.formattedClipRange, - ), - if (stream.clipStartTime != null && stream.clipStartTime! > 0) - _buildMetaInfo( - icon: Icons.play_circle_outline_rounded, - label: 'Start Offset', - value: '${stream.clipStartTime!.toStringAsFixed(1)}s', - ), - _buildMetaInfo( - icon: Icons.calendar_today_outlined, - label: 'Created', - value: stream.formattedDate, - ), - _buildMetaInfo( - icon: Icons.share_outlined, - label: 'Shares', - value: '${stream.shareCounter}', - ), - _buildMetaInfo( - icon: Icons.download_outlined, - label: 'Downloads', - value: '${stream.downloadCounter}', - ), - ], - ), - - const SizedBox(height: 10), - - // Actions Row - Wrap( - alignment: WrapAlignment.spaceBetween, - crossAxisAlignment: WrapCrossAlignment.center, - spacing: 8, - runSpacing: 4, - children: [ - if (stream.onDemandUrl != null && stream.onDemandUrl!.isNotEmpty) - InkWell( - borderRadius: BorderRadius.circular(6), - onTap: () { - Clipboard.setData(ClipboardData(text: stream.onDemandUrl!)); - ScaffoldMessenger.of(context).showSnackBar( - const SnackBar( - content: Text('Stream URL copied to clipboard!'), - duration: Duration(seconds: 2), - behavior: SnackBarBehavior.floating, - ), - ); - }, - child: Padding( - padding: const EdgeInsets.symmetric(horizontal: 6, vertical: 4), - child: Row( - mainAxisSize: MainAxisSize.min, - children: [ - Icon(Icons.copy_rounded, size: 13, color: theme.hintColor), - const SizedBox(width: 4), - Text( - 'Copy Link', - style: TextStyle( - fontSize: 11, - color: theme.hintColor, - fontWeight: FontWeight.w500, - ), - ), - ], - ), - ), - ) - else - const SizedBox.shrink(), - Row( - mainAxisSize: MainAxisSize.min, - children: [ - if (stream.videoId != null) ...[ - TextButton.icon( - onPressed: () => _showStreamDetailsModal(context, stream, initialTab: 1), - icon: const Icon(Icons.bolt_rounded, size: 16, color: Colors.amber), - label: const Text('Moments', style: TextStyle(fontSize: 12, color: Colors.amber)), - style: TextButton.styleFrom( - visualDensity: VisualDensity.compact, - padding: const EdgeInsets.symmetric(horizontal: 8, vertical: 4), - ), - ), - const SizedBox(width: 4), - ], - TextButton.icon( - onPressed: () => _showStreamDetailsModal(context, stream), - icon: const Icon(Icons.info_outline_rounded, size: 16), - label: const Text('Stream Details', style: TextStyle(fontSize: 12)), - style: TextButton.styleFrom( - visualDensity: VisualDensity.compact, - padding: const EdgeInsets.symmetric(horizontal: 10, vertical: 4), - ), - ), - ], - ), - ], - ), - ], - ), - ), - ); - } - - Widget _buildMetaInfo({ - required IconData icon, - required String label, - required String value, - }) { - final theme = Theme.of(context); - return Row( - mainAxisSize: MainAxisSize.min, - children: [ - Icon(icon, size: 14, color: theme.hintColor), - const SizedBox(width: 5), - Text( - '$label: ', - style: TextStyle(fontSize: 12, color: theme.hintColor), - ), - Text( - value, - style: const TextStyle(fontSize: 12, fontWeight: FontWeight.w600), - ), - ], - ); - } - - Widget _buildPaginationBar(BuildContext context) { - final theme = Theme.of(context); - final isDark = theme.brightness == Brightness.dark; - - final startItem = _totalCount == 0 ? 0 : (_currentPage - 1) * _pageSize + 1; - final endItem = (_currentPage * _pageSize).clamp(0, _totalCount); - - return Container( - padding: const EdgeInsets.symmetric(horizontal: 16, vertical: 10), - decoration: BoxDecoration( - color: isDark ? const Color(0xFF1A1A1A) : Colors.white, - border: Border( - top: BorderSide( - color: isDark ? Colors.white10 : Colors.black.withValues(alpha: 0.08), - ), - ), - ), - child: SafeArea( - top: false, - child: Column( - mainAxisSize: MainAxisSize.min, - children: [ - // Top Summary Row: Summary Text & Per-Page Dropdown - Row( - mainAxisAlignment: MainAxisAlignment.spaceBetween, - children: [ - Flexible( - child: Text( - 'Showing $startItem–$endItem of $_totalCount streams', - style: TextStyle( - fontSize: 12, - fontWeight: FontWeight.w500, - color: theme.hintColor, - ), - overflow: TextOverflow.ellipsis, - ), - ), - const SizedBox(width: 8), - Row( - mainAxisSize: MainAxisSize.min, - children: [ - Text( - 'Per page:', - style: TextStyle( - fontSize: 12, - color: theme.hintColor, - ), - ), - const SizedBox(width: 4), - DropdownButtonHideUnderline( - child: DropdownButton( - value: _pageSize, - isDense: true, - style: TextStyle( - fontSize: 12, - fontWeight: FontWeight.w600, - color: theme.colorScheme.primary, - ), - items: const [ - DropdownMenuItem(value: 10, child: Text('10')), - DropdownMenuItem(value: 25, child: Text('25')), - DropdownMenuItem(value: 50, child: Text('50')), - ], - onChanged: (val) { - if (val != null && val != _pageSize) { - setState(() { - _pageSize = val; - }); - _fetchStreams(resetPage: true); - } - }, - ), - ), - ], - ), - ], - ), - const SizedBox(height: 6), - - // Bottom Navigation Row: Centered Pagination Controls - FittedBox( - fit: BoxFit.scaleDown, - child: Row( - mainAxisAlignment: MainAxisAlignment.center, - children: [ - // First Page Button - IconButton( - icon: const Icon(Icons.first_page_rounded, size: 20), - tooltip: 'First Page', - onPressed: _currentPage > 1 ? () => _goToPage(1) : null, - visualDensity: VisualDensity.compact, - ), - - // Previous Page Button - IconButton( - icon: const Icon(Icons.chevron_left_rounded, size: 20), - tooltip: 'Previous Page', - onPressed: _currentPage > 1 ? () => _goToPage(_currentPage - 1) : null, - visualDensity: VisualDensity.compact, - ), - - const SizedBox(width: 4), - - // Page Number Button (Clickable to jump) - InkWell( - onTap: _totalPages > 1 ? _showJumpToPageDialog : null, - borderRadius: BorderRadius.circular(6), - child: Container( - padding: const EdgeInsets.symmetric(horizontal: 10, vertical: 6), - decoration: BoxDecoration( - color: isDark ? const Color(0xFF2A2A2A) : const Color(0xFFEDF2F7), - borderRadius: BorderRadius.circular(6), - ), - child: Text( - 'Page $_currentPage of $_totalPages', - style: TextStyle( - fontSize: 12, - fontWeight: FontWeight.bold, - color: theme.colorScheme.primary, - ), - ), - ), - ), - - const SizedBox(width: 4), - - // Next Page Button - IconButton( - icon: const Icon(Icons.chevron_right_rounded, size: 20), - tooltip: 'Next Page', - onPressed: _currentPage < _totalPages ? () => _goToPage(_currentPage + 1) : null, - visualDensity: VisualDensity.compact, - ), - - // Last Page Button - IconButton( - icon: const Icon(Icons.last_page_rounded, size: 20), - tooltip: 'Last Page', - onPressed: _currentPage < _totalPages ? () => _goToPage(_totalPages) : null, - visualDensity: VisualDensity.compact, - ), - ], - ), - ), - ], - ), - ), - ); - } - - Color _getVideoTypeColor(String? type) { - switch (type) { - case 'sendObs': - return Colors.indigo; - case 'startFirstVideo': - return Colors.teal; - case 'instantReplay': - return Colors.deepOrange; - case 'nonLive': - return Colors.blueGrey; - default: - return Colors.blue; - } - } - - Color _getStatusColor(String? status) { - switch (status?.toLowerCase()) { - case 'complete': - return Colors.green; - case 'live': - return Colors.blue; - case 'failed': - case 'error': - return Colors.red; - default: - return Colors.orange; - } - } - - void _showStreamDetailsModal(BuildContext context, RallyStream stream, {int initialTab = 0}) { - showModalBottomSheet( - context: context, - isScrollControlled: true, - shape: const RoundedRectangleBorder( - borderRadius: BorderRadius.vertical(top: Radius.circular(20)), - ), - builder: (ctx) { - return DefaultTabController( - length: 2, - initialIndex: initialTab, - child: Container( - height: MediaQuery.of(context).size.height * 0.85, - padding: const EdgeInsets.all(20), - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - Row( - children: [ - const Icon(Icons.play_circle_fill_rounded, color: Color(0xFF1E88E5), size: 24), - const SizedBox(width: 10), - Expanded( - child: Text( - 'Stream #${stream.id} Details', - style: const TextStyle( - fontSize: 18, - fontWeight: FontWeight.bold, - ), - ), - ), - IconButton( - icon: const Icon(Icons.close), - onPressed: () => Navigator.pop(ctx), - ), - ], - ), - const TabBar( - tabs: [ - Tab(text: 'Full Stream Info'), - Tab( - child: Row( - mainAxisSize: MainAxisSize.min, - children: [ - Icon(Icons.bolt_rounded, size: 16, color: Colors.amber), - SizedBox(width: 4), - Text('Detected Moments'), - ], - ), - ), - ], - ), - const SizedBox(height: 12), - Expanded( - child: TabBarView( - children: [ - // Tab 1: Full Stream Details - ListView( - children: [ - if (stream.onDemandUrl != null && stream.onDemandUrl!.isNotEmpty) ...[ - RallyVideoPlayer( - videoUrl: stream.onDemandUrl!, - videoTitle: 'Stream #${stream.id}', - initialStartTime: stream.clipStartTime, - initialEndTime: (stream.clipDuration != null && stream.clipDuration! > 0) - ? ((stream.clipStartTime ?? 0.0) + stream.clipDuration!) - : null, - autoPlay: true, - ), - const SizedBox(height: 16), - ], - _buildDetailItem('ID', '${stream.id}'), - _buildDetailItem('Video ID', stream.videoId?.toString() ?? 'N/A'), - _buildDetailItem('Video Type', stream.videoType ?? 'N/A'), - _buildDetailItem('Clip Status', stream.clipStatus ?? 'N/A'), - _buildDetailItem('Clip Window', stream.formattedClipRange), - _buildDetailItem('Clip Duration', '${stream.clipDuration ?? 0} seconds (${stream.formattedDuration})'), - _buildDetailItem('Clip Start Time', '${stream.clipStartTime ?? 0} seconds'), - _buildDetailItem('Created At', stream.formattedDate), - _buildDetailItem('Updated At', stream.updatedAt?.toString() ?? 'N/A'), - _buildDetailItem('Download Counter', '${stream.downloadCounter}'), - _buildDetailItem('Share Counter', '${stream.shareCounter}'), - _buildDetailItem('On Demand URL', stream.onDemandUrl ?? 'N/A', isUrl: true), - ], - ), - - // Tab 2: Detected Moments / Video Actions - FutureBuilder>( - future: stream.videoId != null - ? _actionRepo.getVideoActionsForVideo( - stream.videoId!, - defaultVideoUrl: stream.onDemandUrl, - defaultStreamId: stream.id, - defaultClipStartTime: stream.clipStartTime, - defaultClipDuration: stream.clipDuration, - ) - : Future.value([]), - builder: (context, snapshot) { - if (snapshot.connectionState == ConnectionState.waiting) { - return const Center( - child: Column( - mainAxisAlignment: MainAxisAlignment.center, - children: [ - CircularProgressIndicator(), - SizedBox(height: 12), - Text('Loading detected action moments...'), - ], - ), - ); - } - - if (snapshot.hasError) { - return Center( - child: Text( - 'Error loading moments: ${snapshot.error}', - style: const TextStyle(color: Colors.red), - ), - ); - } - - final actions = snapshot.data ?? []; - if (actions.isEmpty) { - return Center( - child: Column( - mainAxisAlignment: MainAxisAlignment.center, - children: [ - const Icon(Icons.bolt_outlined, size: 48, color: Colors.grey), - const SizedBox(height: 12), - const Text( - 'No action moments detected for this video yet', - style: TextStyle(fontWeight: FontWeight.bold), - ), - const SizedBox(height: 4), - Text( - 'Actions will appear here when detected for Video #${stream.videoId}', - style: const TextStyle(color: Colors.grey, fontSize: 12), - ), - ], - ), - ); - } - - return ListView.builder( - itemCount: actions.length, - itemBuilder: (context, index) { - final action = actions[index]; - return VideoActionCard( - action: action, - onPlay: (act) { - ActionPlayerModal.show(context, act); - }, - ); - }, - ); - }, - ), - ], - ), - ), - ], - ), - ), - ); - }, - ); - } - - Widget _buildDetailItem(String title, String value, {bool isUrl = false}) { - return Padding( - padding: const EdgeInsets.symmetric(vertical: 8.0), - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - Text( - title, - style: const TextStyle( - fontWeight: FontWeight.bold, - fontSize: 12, - color: Colors.grey, - ), - ), - const SizedBox(height: 2), - SelectableText( - value, - style: TextStyle( - fontSize: 14, - fontFamily: isUrl ? 'monospace' : null, - color: isUrl ? const Color(0xFF1E88E5) : null, - ), - ), - ], - ), - ); - } -} diff --git a/lib/screens/video_action_search_screen.dart b/lib/screens/video_action_search_screen.dart deleted file mode 100644 index f674fef..0000000 --- a/lib/screens/video_action_search_screen.dart +++ /dev/null @@ -1,650 +0,0 @@ -import 'package:flutter/material.dart'; -import '../models/video_action.dart'; -import '../models/video_action_search_query.dart'; -import '../services/video_action_repository.dart'; -import '../widgets/action_player_modal.dart'; -import '../widgets/video_action_card.dart'; - -class VideoActionSearchScreen extends StatefulWidget { - final VideoActionSearchQuery? initialQuery; - - const VideoActionSearchScreen({super.key, this.initialQuery}); - - @override - State createState() => _VideoActionSearchScreenState(); -} - -class _VideoActionSearchScreenState extends State { - final VideoActionRepository _repository = VideoActionRepository(); - - // Search input controllers - final TextEditingController _keywordController = TextEditingController(); - final TextEditingController _stageNameController = TextEditingController(); - final TextEditingController _stageNumberController = TextEditingController(); - - String _selectedActionType = 'ALL'; - String _selectedCountry = 'ALL'; - - // Pagination state - int _currentPage = 1; - final int _pageSize = 20; - int _totalCount = 0; - - bool _isLoading = false; - String? _errorMessage; - List _actions = []; - - final List _actionOptions = [ - 'ALL', - 'Jump', - 'Drift', - 'Crash', - 'Spin', - 'Start Line', - 'Near Miss', - 'Mechanical Failure', - 'Offroad', - 'Stuck', - ]; - - final List> _countryOptions = [ - {'label': 'All Countries', 'value': 'ALL'}, - {'label': 'Austria (AT)', 'value': 'Austria'}, - {'label': 'United Kingdom (GB / UK)', 'value': 'United Kingdom'}, - {'label': 'Ireland (IE)', 'value': 'Ireland'}, - {'label': 'Portugal (PT)', 'value': 'Portugal'}, - {'label': 'France (FR)', 'value': 'France'}, - {'label': 'Norway (NO)', 'value': 'Norway'}, - {'label': 'Poland (PL)', 'value': 'Poland'}, - {'label': 'Belgium (BE)', 'value': 'Belgium'}, - {'label': 'Spain (ES)', 'value': 'Spain'}, - {'label': 'Italy (IT)', 'value': 'Italy'}, - {'label': 'Latvia (LV)', 'value': 'Latvia'}, - {'label': 'Czech Republic (CZ)', 'value': 'Czech Republic'}, - {'label': 'Germany (DE)', 'value': 'Germany'}, - {'label': 'Kenya (KE)', 'value': 'Kenya'}, - {'label': 'Croatia (HR)', 'value': 'Croatia'}, - {'label': 'Netherlands (NL)', 'value': 'Netherlands'}, - {'label': 'New Zealand (NZ)', 'value': 'New Zealand'}, - {'label': 'Lithuania (LT)', 'value': 'Lithuania'}, - {'label': 'Slovakia (SK)', 'value': 'Slovakia'}, - {'label': 'Qatar (QA)', 'value': 'Qatar'}, - {'label': 'Pakistan (PK)', 'value': 'Pakistan'}, - {'label': 'Barbados (BB)', 'value': 'Barbados'}, - ]; - - bool _showAdvancedFilters = false; - - @override - void initState() { - super.initState(); - if (widget.initialQuery != null) { - final q = widget.initialQuery!; - if (q.actionType != null) { - _selectedActionType = _formatActionName(q.actionType!); - } - if (q.country != null) { - _selectedCountry = q.country!; - } - if (q.eventName != null) { - _keywordController.text = q.eventName!; - } - if (q.stageName != null) { - _stageNameController.text = q.stageName!; - } - if (q.stageNumber != null) { - _stageNumberController.text = q.stageNumber!; - } - } - _executeSearch(resetPage: true); - } - - @override - void dispose() { - _keywordController.dispose(); - _stageNameController.dispose(); - _stageNumberController.dispose(); - super.dispose(); - } - - String _formatActionName(String raw) { - for (final opt in _actionOptions) { - if (opt.toLowerCase() == raw.toLowerCase() || - '${opt.toLowerCase()}_segments' == raw.toLowerCase()) { - return opt; - } - } - return raw; - } - - VideoActionSearchQuery _buildCurrentQuery({int page = 1}) { - final offset = (page - 1) * _pageSize; - final rawAction = _selectedActionType == 'ALL' ? null : _selectedActionType.toLowerCase(); - final rawCountry = _selectedCountry == 'ALL' ? null : _selectedCountry; - final rawEvent = _keywordController.text.trim().isEmpty ? null : _keywordController.text.trim(); - final rawStageName = _stageNameController.text.trim().isEmpty ? null : _stageNameController.text.trim(); - final rawStageNum = _stageNumberController.text.trim().isEmpty ? null : _stageNumberController.text.trim(); - - return VideoActionSearchQuery( - actionType: rawAction, - country: rawCountry, - eventName: rawEvent, - stageName: rawStageName, - stageNumber: rawStageNum, - limit: _pageSize, - offset: offset, - ); - } - - Future _executeSearch({bool resetPage = false}) async { - if (resetPage) { - _currentPage = 1; - } - - setState(() { - _isLoading = true; - _errorMessage = null; - }); - - try { - final query = _buildCurrentQuery(page: _currentPage); - final count = await _repository.countVideoActions(query); - final results = await _repository.searchVideoActions(query); - - if (mounted) { - setState(() { - _totalCount = count; - _actions = results; - _isLoading = false; - }); - } - } catch (e) { - if (mounted) { - setState(() { - _errorMessage = 'Search failed: $e'; - _isLoading = false; - }); - } - } - } - - void _clearFilters() { - setState(() { - _keywordController.clear(); - _stageNameController.clear(); - _stageNumberController.clear(); - _selectedActionType = 'ALL'; - _selectedCountry = 'ALL'; - }); - _executeSearch(resetPage: true); - } - - @override - Widget build(BuildContext context) { - final theme = Theme.of(context); - final isDark = theme.brightness == Brightness.dark; - final totalPages = (_totalCount / _pageSize).ceil(); - - return Scaffold( - appBar: AppBar( - title: const Row( - children: [ - Icon(Icons.manage_search_rounded, color: Color(0xFF1E88E5)), - SizedBox(width: 8), - Flexible( - child: Text( - 'Action Moments Search', - style: TextStyle(fontWeight: FontWeight.bold), - overflow: TextOverflow.ellipsis, - ), - ), - ], - ), - actions: [ - IconButton( - tooltip: 'Clear All Filters', - icon: const Icon(Icons.filter_alt_off_rounded), - onPressed: _clearFilters, - ), - IconButton( - tooltip: 'Refresh', - icon: const Icon(Icons.refresh_rounded), - onPressed: () => _executeSearch(), - ), - ], - ), - body: Column( - children: [ - // Filter card - Container( - padding: const EdgeInsets.symmetric(horizontal: 16, vertical: 12), - decoration: BoxDecoration( - color: isDark ? const Color(0xFF1E1E1E) : Colors.grey.shade50, - border: Border( - bottom: BorderSide( - color: isDark ? Colors.white12 : Colors.black12, - ), - ), - ), - child: Column( - crossAxisAlignment: CrossAxisAlignment.stretch, - children: [ - // Keyword / Event Name Search - TextField( - controller: _keywordController, - decoration: InputDecoration( - hintText: 'Search by Rally Event (e.g. Trackrod, OBM Land)...', - prefixIcon: const Icon(Icons.search_rounded), - suffixIcon: _keywordController.text.isNotEmpty - ? IconButton( - icon: const Icon(Icons.clear_rounded, size: 20), - onPressed: () { - _keywordController.clear(); - _executeSearch(resetPage: true); - }, - ) - : null, - filled: true, - fillColor: isDark ? const Color(0xFF2A2A2A) : Colors.white, - contentPadding: const EdgeInsets.symmetric(horizontal: 16, vertical: 12), - border: OutlineInputBorder( - borderRadius: BorderRadius.circular(12), - borderSide: BorderSide( - color: isDark ? Colors.white24 : Colors.grey.shade300, - ), - ), - ), - onSubmitted: (_) => _executeSearch(resetPage: true), - ), - const SizedBox(height: 10), - - // Action type & Country dropdowns - Row( - children: [ - // Action dropdown - Expanded( - flex: 5, - child: DropdownButtonFormField( - value: _selectedActionType, - isExpanded: true, - decoration: InputDecoration( - labelText: 'Action Type', - prefixIcon: const Icon(Icons.bolt_rounded, size: 20), - filled: true, - fillColor: isDark ? const Color(0xFF2A2A2A) : Colors.white, - contentPadding: const EdgeInsets.symmetric(horizontal: 12, vertical: 8), - border: OutlineInputBorder( - borderRadius: BorderRadius.circular(12), - borderSide: BorderSide( - color: isDark ? Colors.white24 : Colors.grey.shade300, - ), - ), - ), - items: _actionOptions.map((type) { - return DropdownMenuItem( - value: type, - child: Text( - type == 'ALL' ? 'All Actions' : type, - style: TextStyle( - fontWeight: type == 'ALL' ? FontWeight.normal : FontWeight.w600, - ), - overflow: TextOverflow.ellipsis, - ), - ); - }).toList(), - onChanged: (val) { - if (val != null) { - setState(() => _selectedActionType = val); - _executeSearch(resetPage: true); - } - }, - ), - ), - const SizedBox(width: 8), - - // Country dropdown - Expanded( - flex: 6, - child: DropdownButtonFormField( - value: _selectedCountry, - isExpanded: true, - decoration: InputDecoration( - labelText: 'Country / Location', - prefixIcon: const Icon(Icons.public_rounded, size: 20), - filled: true, - fillColor: isDark ? const Color(0xFF2A2A2A) : Colors.white, - contentPadding: const EdgeInsets.symmetric(horizontal: 12, vertical: 8), - border: OutlineInputBorder( - borderRadius: BorderRadius.circular(12), - borderSide: BorderSide( - color: isDark ? Colors.white24 : Colors.grey.shade300, - ), - ), - ), - items: _countryOptions.map((country) { - return DropdownMenuItem( - value: country['value'], - child: Text( - country['label']!, - overflow: TextOverflow.ellipsis, - style: const TextStyle(fontSize: 13), - ), - ); - }).toList(), - onChanged: (val) { - if (val != null) { - setState(() => _selectedCountry = val); - _executeSearch(resetPage: true); - } - }, - ), - ), - ], - ), - - // Advanced filters expander - if (_showAdvancedFilters) ...[ - const SizedBox(height: 10), - Row( - children: [ - Expanded( - flex: 6, - child: TextField( - controller: _stageNameController, - decoration: InputDecoration( - labelText: 'Stage Name (e.g. Gale Rigg)', - prefixIcon: const Icon(Icons.map_rounded, size: 18), - filled: true, - fillColor: isDark ? const Color(0xFF2A2A2A) : Colors.white, - contentPadding: const EdgeInsets.symmetric(horizontal: 12, vertical: 10), - border: OutlineInputBorder( - borderRadius: BorderRadius.circular(12), - borderSide: BorderSide( - color: isDark ? Colors.white24 : Colors.grey.shade300, - ), - ), - ), - onSubmitted: (_) => _executeSearch(resetPage: true), - ), - ), - const SizedBox(width: 8), - Expanded( - flex: 4, - child: TextField( - controller: _stageNumberController, - decoration: InputDecoration( - labelText: 'Stage # (e.g. 3)', - prefixIcon: const Icon(Icons.numbers_rounded, size: 18), - filled: true, - fillColor: isDark ? const Color(0xFF2A2A2A) : Colors.white, - contentPadding: const EdgeInsets.symmetric(horizontal: 12, vertical: 10), - border: OutlineInputBorder( - borderRadius: BorderRadius.circular(12), - borderSide: BorderSide( - color: isDark ? Colors.white24 : Colors.grey.shade300, - ), - ), - ), - onSubmitted: (_) => _executeSearch(resetPage: true), - ), - ), - ], - ), - ], - - const SizedBox(height: 10), - Row( - children: [ - InkWell( - onTap: () { - setState(() { - _showAdvancedFilters = !_showAdvancedFilters; - }); - }, - borderRadius: BorderRadius.circular(8), - child: Padding( - padding: const EdgeInsets.symmetric(horizontal: 6, vertical: 4), - child: Row( - children: [ - Icon( - _showAdvancedFilters - ? Icons.keyboard_arrow_up_rounded - : Icons.keyboard_arrow_down_rounded, - size: 18, - color: theme.colorScheme.primary, - ), - const SizedBox(width: 4), - Text( - _showAdvancedFilters ? 'Less Filters' : 'Stage Filters', - style: TextStyle( - color: theme.colorScheme.primary, - fontSize: 13, - fontWeight: FontWeight.w600, - ), - ), - ], - ), - ), - ), - const Spacer(), - FilledButton.icon( - onPressed: () => _executeSearch(resetPage: true), - icon: const Icon(Icons.search_rounded, size: 18), - label: const Text('Search'), - style: FilledButton.styleFrom( - padding: const EdgeInsets.symmetric(horizontal: 20, vertical: 10), - shape: RoundedRectangleBorder( - borderRadius: BorderRadius.circular(10), - ), - ), - ), - ], - ), - ], - ), - ), - - // Results count bar - Container( - padding: const EdgeInsets.symmetric(horizontal: 16, vertical: 8), - color: isDark ? const Color(0xFF141414) : Colors.grey.shade100, - child: Row( - mainAxisAlignment: MainAxisAlignment.spaceBetween, - children: [ - Flexible( - child: RichText( - text: TextSpan( - style: TextStyle( - fontSize: 13, - color: isDark ? Colors.white70 : Colors.black54, - ), - children: [ - const TextSpan(text: 'Found '), - TextSpan( - text: '$_totalCount', - style: TextStyle( - fontWeight: FontWeight.bold, - color: theme.colorScheme.primary, - ), - ), - const TextSpan(text: ' action moments'), - ], - ), - overflow: TextOverflow.ellipsis, - ), - ), - if (_totalCount > 0) ...[ - const SizedBox(width: 8), - Text( - 'Page $_currentPage of ${totalPages == 0 ? 1 : totalPages}', - style: TextStyle( - fontSize: 12, - fontWeight: FontWeight.w500, - color: isDark ? Colors.white60 : Colors.black54, - ), - ), - ], - ], - ), - ), - - // Search results - Expanded( - child: _buildResultsView(context), - ), - - // Pagination controls footer - if (_totalCount > _pageSize) _buildPaginationFooter(totalPages), - ], - ), - ); - } - - Widget _buildResultsView(BuildContext context) { - if (_isLoading) { - return const Center( - child: Column( - mainAxisSize: MainAxisSize.min, - children: [ - CircularProgressIndicator(), - SizedBox(height: 16), - Text('Searching action moments...'), - ], - ), - ); - } - - if (_errorMessage != null) { - return Center( - child: Padding( - padding: const EdgeInsets.all(24.0), - child: Column( - mainAxisSize: MainAxisSize.min, - children: [ - const Icon(Icons.error_outline_rounded, color: Colors.red, size: 48), - const SizedBox(height: 12), - Text( - _errorMessage!, - textAlign: TextAlign.center, - style: const TextStyle(color: Colors.red), - ), - const SizedBox(height: 16), - ElevatedButton.icon( - onPressed: () => _executeSearch(), - icon: const Icon(Icons.refresh_rounded), - label: const Text('Retry'), - ), - ], - ), - ), - ); - } - - if (_actions.isEmpty) { - return Center( - child: Padding( - padding: const EdgeInsets.all(32.0), - child: Column( - mainAxisSize: MainAxisSize.min, - children: [ - Icon( - Icons.search_off_rounded, - size: 64, - color: Colors.grey.withValues(alpha: 0.4), - ), - const SizedBox(height: 16), - const Text( - 'No action moments found', - style: TextStyle(fontSize: 18, fontWeight: FontWeight.bold), - ), - const SizedBox(height: 8), - const Text( - 'Try broadening your search filters or resetting all filters.', - textAlign: TextAlign.center, - style: TextStyle(color: Colors.grey), - ), - const SizedBox(height: 20), - OutlinedButton.icon( - onPressed: _clearFilters, - icon: const Icon(Icons.clear_all_rounded), - label: const Text('Reset Filters'), - ), - ], - ), - ), - ); - } - - return ListView.builder( - padding: const EdgeInsets.all(12), - itemCount: _actions.length, - itemBuilder: (context, index) { - final action = _actions[index]; - return VideoActionCard( - action: action, - onPlay: (act) => ActionPlayerModal.show(context, act), - ); - }, - ); - } - - Widget _buildPaginationFooter(int totalPages) { - final theme = Theme.of(context); - final isDark = theme.brightness == Brightness.dark; - - return Container( - padding: const EdgeInsets.symmetric(horizontal: 12, vertical: 8), - decoration: BoxDecoration( - color: theme.cardColor, - border: Border( - top: BorderSide( - color: isDark ? Colors.white12 : Colors.black12, - ), - ), - ), - child: SafeArea( - top: false, - child: Row( - children: [ - OutlinedButton.icon( - onPressed: _currentPage > 1 && !_isLoading - ? () { - setState(() => _currentPage--); - _executeSearch(); - } - : null, - icon: const Icon(Icons.chevron_left_rounded, size: 18), - label: const Text('Prev'), - style: OutlinedButton.styleFrom( - padding: const EdgeInsets.symmetric(horizontal: 10, vertical: 6), - visualDensity: VisualDensity.compact, - ), - ), - Expanded( - child: Text( - 'Page $_currentPage of $totalPages', - textAlign: TextAlign.center, - style: const TextStyle(fontWeight: FontWeight.w600, fontSize: 12), - overflow: TextOverflow.ellipsis, - ), - ), - OutlinedButton.icon( - onPressed: _currentPage < totalPages && !_isLoading - ? () { - setState(() => _currentPage++); - _executeSearch(); - } - : null, - icon: const Icon(Icons.chevron_right_rounded, size: 18), - label: const Text('Next'), - style: OutlinedButton.styleFrom( - padding: const EdgeInsets.symmetric(horizontal: 10, vertical: 6), - visualDensity: VisualDensity.compact, - ), - ), - ], - ), - ), - ); - } -} diff --git a/lib/services/database_service.dart b/lib/services/database_service.dart deleted file mode 100644 index 1673f1b..0000000 --- a/lib/services/database_service.dart +++ /dev/null @@ -1,1429 +0,0 @@ -import 'dart:developer' as developer; -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:mysql_client/mysql_client.dart'; -import '../models/search_intent.dart'; -import '../models/search_query.dart'; -import '../models/video_action_search_query.dart'; - -class DatabaseService { - static final DatabaseService _instance = DatabaseService._internal(); - factory DatabaseService() => _instance; - DatabaseService._internal(); - - MySQLConnection? _connection; - - String get host => dotenv.env['DB_HOST'] ?? ''; - int get port => int.tryParse(dotenv.env['DB_PORT'] ?? '3306') ?? 3306; - String get databaseName => dotenv.env['DB_NAME'] ?? ''; - String get userName => dotenv.env['DB_USER'] ?? ''; - String get password => dotenv.env['DB_PASSWORD'] ?? ''; - bool get isSecure => - (dotenv.env['DB_USE_SSL'] ?? 'false').toLowerCase() == 'true'; - - bool get isConnected => _connection != null && _connection!.connected; - - /// Connects to the AWS RDS MySQL database - Future connect() async { - if (_connection != null && _connection!.connected) { - return _connection!; - } - - try { - developer.log( - 'Connecting to MySQL: $host:$port/$databaseName as $userName', - name: 'DatabaseService', - ); - - _connection = await MySQLConnection.createConnection( - host: host, - port: port, - userName: userName, - password: password, - databaseName: databaseName, - secure: isSecure, - ); - - await _connection!.connect(); - developer.log('MySQL connection established successfully.', - name: 'DatabaseService'); - return _connection!; - } catch (e, st) { - developer.log('Failed to connect to MySQL database', - name: 'DatabaseService', error: e, stackTrace: st); - rethrow; - } - } - - /// Tests the database connection and returns connection info & table count - Future> testConnection() async { - final stopwatch = Stopwatch()..start(); - try { - final conn = await connect(); - final result = await conn.execute('SHOW TABLES;'); - stopwatch.stop(); - - final tables = result.rows - .map((row) => row.assoc().values.first?.toString() ?? '') - .toList(); - - return { - 'success': true, - 'latencyMs': stopwatch.elapsedMilliseconds, - 'host': host, - 'database': databaseName, - 'tableCount': tables.length, - 'tables': tables, - }; - } catch (e) { - stopwatch.stop(); - return { - 'success': false, - 'latencyMs': stopwatch.elapsedMilliseconds, - 'error': e.toString(), - }; - } - } - - /// Executes a query and returns the results as a `List>` - Future>> query( - String sql, [ - Map? params, - ]) async { - final conn = await connect(); - final result = await conn.execute(sql, params ?? {}); - return result.rows.map((row) => row.assoc()).toList(); - } - - /// Fetches a paginated list of rally streams with optional filters and sorting - Future>> getRallyStreams({ - int limit = 10, - int offset = 0, - String? searchQuery, - String? videoType, - String? clipStatus, - String sortBy = 'id', - bool sortAscending = false, - }) async { - final whereClauses = [ - "(video_type IS NULL OR video_type != 'instantReplay')" - ]; - - if (searchQuery != null && searchQuery.trim().isNotEmpty) { - final sanitized = searchQuery.trim().replaceAll("'", "''"); - final isNum = int.tryParse(sanitized) != null; - if (isNum) { - whereClauses.add('(id = $sanitized OR video_id = $sanitized OR on_demand_url LIKE \'%$sanitized%\')'); - } else { - whereClauses.add('(on_demand_url LIKE \'%$sanitized%\' OR video_type LIKE \'%$sanitized%\' OR clip_status LIKE \'%$sanitized%\')'); - } - } - - if (videoType != null && videoType.isNotEmpty && videoType.toLowerCase() != 'all') { - final sanitizedType = videoType.replaceAll("'", "''"); - whereClauses.add('video_type = \'$sanitizedType\''); - } - - if (clipStatus != null && clipStatus.isNotEmpty && clipStatus.toLowerCase() != 'all') { - final sanitizedStatus = clipStatus.replaceAll("'", "''"); - whereClauses.add('clip_status = \'$sanitizedStatus\''); - } - - final whereSql = whereClauses.isNotEmpty ? 'WHERE ${whereClauses.join(' AND ')}' : ''; - final orderDirection = sortAscending ? 'ASC' : 'DESC'; - final sql = 'SELECT * FROM `rally_streams` $whereSql ORDER BY `$sortBy` $orderDirection LIMIT $limit OFFSET $offset;'; - - return await query(sql); - } - - /// Returns the total count of rally streams matching the filter - Future getRallyStreamsCount({ - String? searchQuery, - String? videoType, - String? clipStatus, - }) async { - final whereClauses = [ - "(video_type IS NULL OR video_type != 'instantReplay')" - ]; - - if (searchQuery != null && searchQuery.trim().isNotEmpty) { - final sanitized = searchQuery.trim().replaceAll("'", "''"); - final isNum = int.tryParse(sanitized) != null; - if (isNum) { - whereClauses.add('(id = $sanitized OR video_id = $sanitized OR on_demand_url LIKE \'%$sanitized%\')'); - } else { - whereClauses.add('(on_demand_url LIKE \'%$sanitized%\' OR video_type LIKE \'%$sanitized%\' OR clip_status LIKE \'%$sanitized%\')'); - } - } - - if (videoType != null && videoType.isNotEmpty && videoType.toLowerCase() != 'all') { - final sanitizedType = videoType.replaceAll("'", "''"); - whereClauses.add('video_type = \'$sanitizedType\''); - } - - if (clipStatus != null && clipStatus.isNotEmpty && clipStatus.toLowerCase() != 'all') { - final sanitizedStatus = clipStatus.replaceAll("'", "''"); - whereClauses.add('clip_status = \'$sanitizedStatus\''); - } - - final whereSql = whereClauses.isNotEmpty ? 'WHERE ${whereClauses.join(' AND ')}' : ''; - final sql = 'SELECT COUNT(*) as count FROM `rally_streams` $whereSql;'; - - final result = await query(sql); - if (result.isNotEmpty) { - final countVal = result.first['count']; - if (countVal is int) return countVal; - return int.tryParse(countVal.toString()) ?? 0; - } - return 0; - } - - /// Fetches actions / moments for a specific source video ID - Future>> getVideoActionsForVideo(int videoId) async { - final sql = ''' - SELECT - vm.id AS id, - vm.video_id AS video_id, - rs.id AS stream_id, - rs.on_demand_url AS on_demand_url, - rs.clip_start_time AS clip_start_time, - rs.clip_duration AS clip_duration, - va.id AS action_type_id, - va.action_name AS action_name, - vm.start_action AS start_action, - vm.end_action AS end_action, - vm.points AS points, - rv.thumbnail AS thumbnail_url, - stg.stage_name, - stg.stage_number, - ev.event_name, - ev.country AS event_country - FROM rally_video_metadata vm - INNER JOIN rally_video_actions va ON vm.action_id = va.id - LEFT JOIN rally_streams rs ON vm.video_id = rs.video_id AND (rs.video_type IS NULL OR rs.video_type != 'instantReplay') - LEFT JOIN rally_videos rv ON vm.video_id = rv.id - LEFT JOIN rally_stages stg ON rv.stage_id = stg.stage_id - LEFT JOIN rally_events ev ON stg.event_id = ev.event_id - WHERE vm.video_id = $videoId - ORDER BY vm.start_action ASC; - '''; - - return await query(sql); - } - - /// Fetches actions / moments for a specific rally stream ID - Future>> getVideoActionsForStream(int streamId) async { - final sql = ''' - SELECT - vm.id AS id, - vm.video_id AS video_id, - rs.id AS stream_id, - rs.on_demand_url AS on_demand_url, - rs.clip_start_time AS clip_start_time, - rs.clip_duration AS clip_duration, - va.id AS action_type_id, - va.action_name AS action_name, - vm.start_action AS start_action, - vm.end_action AS end_action, - vm.points AS points, - rv.thumbnail AS thumbnail_url, - stg.stage_name, - stg.stage_number, - ev.event_name, - ev.country AS event_country - FROM rally_streams rs - INNER JOIN rally_video_metadata vm ON rs.video_id = vm.video_id - INNER JOIN rally_video_actions va ON vm.action_id = va.id - LEFT JOIN rally_videos rv ON rs.video_id = rv.id - LEFT JOIN rally_stages stg ON rv.stage_id = stg.stage_id - LEFT JOIN rally_events ev ON stg.event_id = ev.event_id - WHERE rs.id = $streamId - ORDER BY vm.start_action ASC; - '''; - - return await query(sql); - } - - /// Fetches recent action moments across all streams - Future>> getRecentVideoActions({ - int limit = 20, - int offset = 0, - String? actionType, - }) async { - final whereClauses = [ - "rs.on_demand_url IS NOT NULL AND rs.on_demand_url != ''", - "(rs.video_type IS NULL OR rs.video_type != 'instantReplay')" - ]; - - if (actionType != null && actionType.isNotEmpty && actionType.toLowerCase() != 'all') { - final sanitizedType = actionType.replaceAll("'", "''"); - whereClauses.add("(va.action_name = '$sanitizedType' OR va.action_name = '${sanitizedType}_segments')"); - } - - final whereSql = 'WHERE ${whereClauses.join(' AND ')}'; - final sql = ''' - SELECT - vm.id AS id, - vm.video_id AS video_id, - rs.id AS stream_id, - rs.on_demand_url AS on_demand_url, - rs.clip_start_time AS clip_start_time, - rs.clip_duration AS clip_duration, - va.id AS action_type_id, - va.action_name AS action_name, - vm.start_action AS start_action, - vm.end_action AS end_action, - vm.points AS points, - rv.thumbnail AS thumbnail_url, - stg.stage_name, - stg.stage_number, - ev.event_name, - ev.country AS event_country - FROM rally_video_metadata vm - INNER JOIN rally_video_actions va ON vm.action_id = va.id - INNER JOIN rally_streams rs ON vm.video_id = rs.video_id - LEFT JOIN rally_videos rv ON vm.video_id = rv.id - LEFT JOIN rally_stages stg ON rv.stage_id = stg.stage_id - LEFT JOIN rally_events ev ON stg.event_id = ev.event_id - $whereSql - ORDER BY vm.id DESC - LIMIT $limit OFFSET $offset; - '''; - - return await query(sql); - } - - // =========================================================================== - // MULTI-VALUE SQL CLAUSE GENERATION HELPERS - // Enforces: OR within one dimension, AND across different dimensions. - // =========================================================================== - - /// Subquery condition that identifies the final stage of each rally event - static const String _finalStageSubquery = ''' - (ev.event_id, CAST(stg.stage_number AS UNSIGNED)) IN ( - SELECT s2.event_id, MAX(CAST(s2.stage_number AS UNSIGNED)) - FROM rally_stages s2 - INNER JOIN rally_results r2 ON s2.stage_id = r2.stage_id AND s2.event_id = r2.rally_id - GROUP BY s2.event_id - ) - '''; - - /// Builds WHERE clauses for countries (OR within dimension) - List _buildCountryWhereClauses(SearchQuery q, {String prefix = 'ev.'}) { - final aliases = q.resolvedCountryAliases; - if (aliases.isEmpty) return []; - - final countryIn = aliases.map((c) => "'${c.replaceAll("'", "''")}'").join(', '); - final likeClauses = []; - for (final c in q.countries) { - final sanitized = c.trim().replaceAll("'", "''").toLowerCase(); - if (sanitized.length > 2) { - likeClauses.add("LOWER(${prefix}country) LIKE '%$sanitized%'"); - } - } - - if (likeClauses.isNotEmpty) { - return ["(LOWER(${prefix}country) IN ($countryIn) OR ${likeClauses.join(' OR ')})"]; - } - return ["LOWER(${prefix}country) IN ($countryIn)"]; - } - - /// Builds WHERE clauses for cities (OR within dimension) - List _buildCityWhereClauses(SearchQuery q, {String prefix = 'ev.'}) { - if (q.cities.isEmpty) return []; - - final cityClauses = []; - for (final city in q.cities) { - if (city.trim().toUpperCase() == 'ALL') continue; - final sanitized = city.trim().replaceAll("'", "''").toLowerCase(); - cityClauses.add("LOWER(${prefix}city) LIKE '%$sanitized%'"); - } - - if (cityClauses.isNotEmpty) { - return ["(${cityClauses.join(' OR ')})"]; - } - return []; - } - - /// Builds WHERE clauses for years and year ranges (OR within dimension) - List _buildYearWhereClauses(SearchQuery q, {String prefix = 'ev.'}) { - final yearClauses = []; - - if (q.years.isNotEmpty) { - final yearsIn = q.years.join(', '); - yearClauses.add("COALESCE(YEAR(${prefix}start_date), YEAR(${prefix}end_date)) IN ($yearsIn)"); - } - - if (q.yearFrom != null && q.yearTo != null) { - yearClauses.add("(COALESCE(YEAR(${prefix}start_date), YEAR(${prefix}end_date)) BETWEEN ${q.yearFrom} AND ${q.yearTo})"); - } else if (q.yearFrom != null) { - yearClauses.add("COALESCE(YEAR(${prefix}start_date), YEAR(${prefix}end_date)) >= ${q.yearFrom}"); - } else if (q.yearTo != null) { - yearClauses.add("COALESCE(YEAR(${prefix}start_date), YEAR(${prefix}end_date)) <= ${q.yearTo}"); - } - - if (yearClauses.isNotEmpty) { - return ["(${yearClauses.join(' OR ')})"]; - } - return []; - } - - /// Builds WHERE clauses for rallies / events (OR within dimension) - List _buildRallyWhereClauses(SearchQuery q, {String prefix = 'ev.'}) { - final names = q.targetRallyNames; - if (names.isEmpty) return []; - - final rallyClauses = []; - for (final r in names) { - final sanitized = r.trim().replaceAll("'", "''").toLowerCase(); - rallyClauses.add("(LOWER(${prefix}event_name) LIKE '%$sanitized%' OR ${prefix}event_id = '$sanitized')"); - } - - if (rallyClauses.isNotEmpty) { - return ["(${rallyClauses.join(' OR ')})"]; - } - return []; - } - - /// Builds WHERE clauses for stages and stage numbers (OR within dimension) - List _buildStageWhereClauses(SearchQuery q, {String prefix = 'stg.'}) { - final clauses = []; - - if (q.stageNames.isNotEmpty) { - final stageClauses = []; - for (final st in q.stageNames) { - final sanitized = st.trim().replaceAll("'", "''").toLowerCase(); - stageClauses.add("LOWER(${prefix}stage_name) LIKE '%$sanitized%'"); - } - if (stageClauses.isNotEmpty) { - clauses.add("(${stageClauses.join(' OR ')})"); - } - } - - if (q.stageNumbers.isNotEmpty) { - final numClauses = []; - for (final sn in q.stageNumbers) { - final sanitized = sn.trim().replaceAll("'", "''").toLowerCase(); - final cleanNum = sanitized.replaceAll('ss', '').trim(); - numClauses.add("(${prefix}stage_number = '$cleanNum' OR ${prefix}stage_number = '$sanitized' OR LOWER(${prefix}stage_name) LIKE '%stage $cleanNum%')"); - } - if (numClauses.isNotEmpty) { - clauses.add("(${numClauses.join(' OR ')})"); - } - } - - return clauses; - } - - /// Builds person / driver / co-driver where clauses respecting the query's PersonRole - List _buildPersonWhereClauses( - SearchQuery q, { - String driverAlias = 'dp', - String codriverAlias = 'cdp', - String entryListAlias = 'el', - }) { - final clauses = []; - - if (q.driverIds.isNotEmpty) { - final idsIn = q.driverIds.map((id) => "'${id.replaceAll("'", "''")}'").join(', '); - switch (q.personRole) { - case PersonRole.driver: - clauses.add("($driverAlias.driver_id IN ($idsIn) OR $entryListAlias.user_driver_id IN ($idsIn))"); - break; - case PersonRole.coDriver: - clauses.add("($codriverAlias.codriver_id IN ($idsIn) OR $entryListAlias.user_co_driver_id IN ($idsIn))"); - break; - case PersonRole.any: - clauses.add("($driverAlias.driver_id IN ($idsIn) OR $entryListAlias.user_driver_id IN ($idsIn) OR $codriverAlias.codriver_id IN ($idsIn) OR $entryListAlias.user_co_driver_id IN ($idsIn))"); - break; - } - } - - for (final d in q.driverNames) { - final sanitized = d.trim().replaceAll("'", "''").toLowerCase(); - final tokens = sanitized.split(' ').where((t) => t.isNotEmpty).toList(); - - final String dMatch; - final String cdMatch; - if (tokens.length > 1) { - final dTokens = tokens.map((t) => "LOWER($driverAlias.full_name) LIKE '%$t%'").join(' AND '); - final cdTokens = tokens.map((t) => "LOWER($codriverAlias.full_name) LIKE '%$t%'").join(' AND '); - dMatch = "($dTokens OR LOWER($driverAlias.nick_name) LIKE '%$sanitized%' OR LOWER($entryListAlias.driver_link) LIKE '%$sanitized%')"; - cdMatch = "($cdTokens OR LOWER($codriverAlias.nick_name) LIKE '%$sanitized%' OR LOWER($entryListAlias.co_driver_link) LIKE '%$sanitized%')"; - } else { - dMatch = "(LOWER($driverAlias.full_name) LIKE '%$sanitized%' OR LOWER($driverAlias.nick_name) LIKE '%$sanitized%' OR LOWER($entryListAlias.driver_link) LIKE '%$sanitized%')"; - cdMatch = "(LOWER($codriverAlias.full_name) LIKE '%$sanitized%' OR LOWER($codriverAlias.nick_name) LIKE '%$sanitized%' OR LOWER($entryListAlias.co_driver_link) LIKE '%$sanitized%')"; - } - - switch (q.personRole) { - case PersonRole.driver: - clauses.add(dMatch); - break; - case PersonRole.coDriver: - clauses.add(cdMatch); - break; - case PersonRole.any: - clauses.add("($dMatch OR $cdMatch)"); - break; - } - } - - return clauses; - } - - // =========================================================================== - // 1. SEARCH VIDEO ACTIONS - // =========================================================================== - - /// Searches video actions deterministically using structured multi-value query filters - Future>> searchVideoActions(dynamic searchQuery) async { - final SearchQuery q = _normalizeSearchQuery(searchQuery); - - final whereClauses = [ - "rs.on_demand_url IS NOT NULL AND rs.on_demand_url != ''", - "(rs.video_type IS NULL OR rs.video_type != 'instantReplay')" - ]; - - // Action types filter (OR within dimension) - final resolvedActions = q.resolvedActionTypes; - if (resolvedActions.isNotEmpty) { - final actionIn = resolvedActions - .map((a) => "'${a.replaceAll("'", "''")}'") - .join(', '); - whereClauses.add("va.action_name IN ($actionIn)"); - } - - whereClauses.addAll(_buildCountryWhereClauses(q)); - whereClauses.addAll(_buildCityWhereClauses(q)); - whereClauses.addAll(_buildYearWhereClauses(q)); - whereClauses.addAll(_buildRallyWhereClauses(q)); - whereClauses.addAll(_buildStageWhereClauses(q)); - - // Person filter (Driver Name / Driver ID / Co-Driver) (OR within dimension) - final driverClauses = _buildPersonWhereClauses(q, driverAlias: 'dp', codriverAlias: 'cdp', entryListAlias: 'el'); - if (driverClauses.isNotEmpty) { - whereClauses.add("(${driverClauses.join(' OR ')})"); - } - - final whereSql = 'WHERE ${whereClauses.join(' AND ')}'; - final sql = ''' - SELECT - vm.id AS id, - vm.video_id AS video_id, - MIN(rs.id) AS stream_id, - MIN(rs.on_demand_url) AS on_demand_url, - MIN(rs.clip_start_time) AS clip_start_time, - MIN(rs.clip_duration) AS clip_duration, - va.id AS action_type_id, - va.action_name AS action_name, - vm.start_action AS start_action, - vm.end_action AS end_action, - vm.points AS points, - rv.thumbnail AS thumbnail_url, - stg.stage_name, - stg.stage_number, - ev.event_name, - ev.country AS event_country, - COALESCE(dp.full_name, cdp.full_name) AS driver_name - FROM rally_video_metadata vm - INNER JOIN rally_video_actions va ON vm.action_id = va.id - INNER JOIN rally_streams rs ON vm.video_id = rs.video_id - LEFT JOIN rally_videos rv ON vm.video_id = rv.id - LEFT JOIN rally_stages stg ON rv.stage_id = stg.stage_id - LEFT JOIN rally_events ev ON stg.event_id = ev.event_id - LEFT JOIN rally_entry_list el ON vm.entry_list_id = el.id - LEFT JOIN user_driver_profile dp ON el.user_driver_id = dp.driver_id - LEFT JOIN user_codriver_profile cdp ON el.user_co_driver_id = cdp.codriver_id - $whereSql - GROUP BY vm.id, vm.video_id, va.id, va.action_name, vm.start_action, vm.end_action, vm.points, rv.thumbnail, stg.stage_name, stg.stage_number, ev.event_name, ev.country, dp.full_name, cdp.full_name - ORDER BY vm.id DESC - LIMIT ${q.limit} OFFSET ${q.offset}; - '''; - - return await query(sql); - } - - /// Returns total count of video actions matching the multi-value search query - Future countVideoActions(dynamic searchQuery) async { - final SearchQuery q = _normalizeSearchQuery(searchQuery); - - final whereClauses = [ - "rs.on_demand_url IS NOT NULL AND rs.on_demand_url != ''", - "(rs.video_type IS NULL OR rs.video_type != 'instantReplay')" - ]; - - final resolvedActions = q.resolvedActionTypes; - if (resolvedActions.isNotEmpty) { - final actionIn = resolvedActions - .map((a) => "'${a.replaceAll("'", "''")}'") - .join(', '); - whereClauses.add("va.action_name IN ($actionIn)"); - } - - whereClauses.addAll(_buildCountryWhereClauses(q)); - whereClauses.addAll(_buildCityWhereClauses(q)); - whereClauses.addAll(_buildYearWhereClauses(q)); - whereClauses.addAll(_buildRallyWhereClauses(q)); - whereClauses.addAll(_buildStageWhereClauses(q)); - - final driverClauses = _buildPersonWhereClauses(q, driverAlias: 'dp', codriverAlias: 'cdp', entryListAlias: 'el'); - if (driverClauses.isNotEmpty) { - whereClauses.add("(${driverClauses.join(' OR ')})"); - } - - final whereSql = 'WHERE ${whereClauses.join(' AND ')}'; - final sql = ''' - SELECT COUNT(DISTINCT vm.id) as count - FROM rally_video_metadata vm - INNER JOIN rally_video_actions va ON vm.action_id = va.id - INNER JOIN rally_streams rs ON vm.video_id = rs.video_id - LEFT JOIN rally_videos rv ON vm.video_id = rv.id - LEFT JOIN rally_stages stg ON rv.stage_id = stg.stage_id - LEFT JOIN rally_events ev ON stg.event_id = ev.event_id - LEFT JOIN rally_entry_list el ON vm.entry_list_id = el.id - LEFT JOIN user_driver_profile dp ON el.user_driver_id = dp.driver_id - LEFT JOIN user_codriver_profile cdp ON el.user_co_driver_id = cdp.codriver_id - $whereSql; - '''; - - final result = await query(sql); - if (result.isNotEmpty) { - final countVal = result.first['count']; - if (countVal is int) return countVal; - return int.tryParse(countVal.toString()) ?? 0; - } - return 0; - } - - // =========================================================================== - // 2. SEARCH RALLIES - // =========================================================================== - - /// Searches rally events by countries, cities, years, drivers/co-drivers, or event names - Future>> searchRallies(SearchQuery q) async { - final whereClauses = []; - - whereClauses.addAll(_buildCountryWhereClauses(q)); - whereClauses.addAll(_buildCityWhereClauses(q)); - whereClauses.addAll(_buildYearWhereClauses(q)); - whereClauses.addAll(_buildRallyWhereClauses(q)); - - // Person (driver/co-driver) participation subqueries via rally_entry_list -> rally_sub_events - if (q.driverMatchMode == MatchMode.all && (q.driverIds.length > 1 || q.driverNames.length > 1)) { - // Explicit ALL semantics: event must contain ALL requested persons - for (final id in q.driverIds) { - final sanitizedId = id.replaceAll("'", "''"); - final roleClause = q.personRole == PersonRole.driver - ? "(dpx.driver_id = '$sanitizedId' OR elx.user_driver_id = '$sanitizedId')" - : (q.personRole == PersonRole.coDriver - ? "(cdpx.codriver_id = '$sanitizedId' OR elx.user_co_driver_id = '$sanitizedId')" - : "(dpx.driver_id = '$sanitizedId' OR elx.user_driver_id = '$sanitizedId' OR cdpx.codriver_id = '$sanitizedId' OR elx.user_co_driver_id = '$sanitizedId')"); - whereClauses.add(''' - ev.event_id IN ( - SELECT DISTINCT sex.event_id - FROM rally_entry_list elx - JOIN rally_sub_events sex ON elx.sub_event_id = sex.sub_event_id - LEFT JOIN user_driver_profile dpx ON elx.user_driver_id = dpx.driver_id - LEFT JOIN user_codriver_profile cdpx ON elx.user_co_driver_id = cdpx.codriver_id - WHERE $roleClause - ) - '''); - } - for (final d in q.driverNames) { - final sanitized = d.trim().replaceAll("'", "''").toLowerCase(); - final roleClause = q.personRole == PersonRole.driver - ? "(LOWER(dpx.full_name) LIKE '%$sanitized%' OR LOWER(dpx.nick_name) LIKE '%$sanitized%' OR LOWER(elx.driver_link) LIKE '%$sanitized%')" - : (q.personRole == PersonRole.coDriver - ? "(LOWER(cdpx.full_name) LIKE '%$sanitized%' OR LOWER(cdpx.nick_name) LIKE '%$sanitized%' OR LOWER(elx.co_driver_link) LIKE '%$sanitized%')" - : "(LOWER(dpx.full_name) LIKE '%$sanitized%' OR LOWER(dpx.nick_name) LIKE '%$sanitized%' OR LOWER(cdpx.full_name) LIKE '%$sanitized%' OR LOWER(cdpx.nick_name) LIKE '%$sanitized%' OR LOWER(elx.driver_link) LIKE '%$sanitized%' OR LOWER(elx.co_driver_link) LIKE '%$sanitized%')"); - whereClauses.add(''' - ev.event_id IN ( - SELECT DISTINCT sex.event_id - FROM rally_entry_list elx - JOIN rally_sub_events sex ON elx.sub_event_id = sex.sub_event_id - LEFT JOIN user_driver_profile dpx ON elx.user_driver_id = dpx.driver_id - LEFT JOIN user_codriver_profile cdpx ON elx.user_co_driver_id = cdpx.codriver_id - WHERE $roleClause - ) - '''); - } - } else { - // Default ANY semantics: event contains ANY requested person - final subClauses = _buildPersonWhereClauses(q, driverAlias: 'dpx', codriverAlias: 'cdpx', entryListAlias: 'elx'); - if (subClauses.isNotEmpty) { - whereClauses.add(''' - ev.event_id IN ( - SELECT DISTINCT sex.event_id - FROM rally_entry_list elx - JOIN rally_sub_events sex ON elx.sub_event_id = sex.sub_event_id - LEFT JOIN user_driver_profile dpx ON elx.user_driver_id = dpx.driver_id - LEFT JOIN user_codriver_profile cdpx ON elx.user_co_driver_id = cdpx.codriver_id - WHERE (${subClauses.join(' OR ')}) - ) - '''); - } - } - - final whereSql = whereClauses.isNotEmpty ? 'WHERE ${whereClauses.join(' AND ')}' : ''; - final sql = ''' - SELECT - ev.event_id, - ev.event_name, - ev.status, - ev.start_date, - ev.end_date, - ev.stages_count, - ev.country, - ev.city, - ev.thumbnail, - ev.logo, - COUNT(DISTINCT stg.stage_id) AS calculated_stages_count - FROM rally_events ev - LEFT JOIN rally_stages stg ON ev.event_id = stg.event_id - $whereSql - GROUP BY ev.event_id, ev.event_name, ev.status, ev.start_date, ev.end_date, ev.stages_count, ev.country, ev.city, ev.thumbnail, ev.logo - ORDER BY ev.start_date DESC - LIMIT ${q.limit} OFFSET ${q.offset}; - '''; - - return await query(sql); - } - - /// Returns total count of rallies matching search criteria - Future countRallies(SearchQuery q) async { - final whereClauses = []; - - whereClauses.addAll(_buildCountryWhereClauses(q)); - whereClauses.addAll(_buildCityWhereClauses(q)); - whereClauses.addAll(_buildYearWhereClauses(q)); - whereClauses.addAll(_buildRallyWhereClauses(q)); - - if (q.driverMatchMode == MatchMode.all && (q.driverIds.length > 1 || q.driverNames.length > 1)) { - for (final id in q.driverIds) { - final sanitizedId = id.replaceAll("'", "''"); - final roleClause = q.personRole == PersonRole.driver - ? "(dpx.driver_id = '$sanitizedId' OR elx.user_driver_id = '$sanitizedId')" - : (q.personRole == PersonRole.coDriver - ? "(cdpx.codriver_id = '$sanitizedId' OR elx.user_co_driver_id = '$sanitizedId')" - : "(dpx.driver_id = '$sanitizedId' OR elx.user_driver_id = '$sanitizedId' OR cdpx.codriver_id = '$sanitizedId' OR elx.user_co_driver_id = '$sanitizedId')"); - whereClauses.add(''' - ev.event_id IN ( - SELECT DISTINCT sex.event_id - FROM rally_entry_list elx - JOIN rally_sub_events sex ON elx.sub_event_id = sex.sub_event_id - LEFT JOIN user_driver_profile dpx ON elx.user_driver_id = dpx.driver_id - LEFT JOIN user_codriver_profile cdpx ON elx.user_co_driver_id = cdpx.codriver_id - WHERE $roleClause - ) - '''); - } - for (final d in q.driverNames) { - final sanitized = d.trim().replaceAll("'", "''").toLowerCase(); - final roleClause = q.personRole == PersonRole.driver - ? "(LOWER(dpx.full_name) LIKE '%$sanitized%' OR LOWER(dpx.nick_name) LIKE '%$sanitized%' OR LOWER(elx.driver_link) LIKE '%$sanitized%')" - : (q.personRole == PersonRole.coDriver - ? "(LOWER(cdpx.full_name) LIKE '%$sanitized%' OR LOWER(cdpx.nick_name) LIKE '%$sanitized%' OR LOWER(elx.co_driver_link) LIKE '%$sanitized%')" - : "(LOWER(dpx.full_name) LIKE '%$sanitized%' OR LOWER(dpx.nick_name) LIKE '%$sanitized%' OR LOWER(cdpx.full_name) LIKE '%$sanitized%' OR LOWER(cdpx.nick_name) LIKE '%$sanitized%' OR LOWER(elx.driver_link) LIKE '%$sanitized%' OR LOWER(elx.co_driver_link) LIKE '%$sanitized%')"); - whereClauses.add(''' - ev.event_id IN ( - SELECT DISTINCT sex.event_id - FROM rally_entry_list elx - JOIN rally_sub_events sex ON elx.sub_event_id = sex.sub_event_id - LEFT JOIN user_driver_profile dpx ON elx.user_driver_id = dpx.driver_id - LEFT JOIN user_codriver_profile cdpx ON elx.user_co_driver_id = cdpx.codriver_id - WHERE $roleClause - ) - '''); - } - } else { - final subClauses = _buildPersonWhereClauses(q, driverAlias: 'dpx', codriverAlias: 'cdpx', entryListAlias: 'elx'); - if (subClauses.isNotEmpty) { - whereClauses.add(''' - ev.event_id IN ( - SELECT DISTINCT sex.event_id - FROM rally_entry_list elx - JOIN rally_sub_events sex ON elx.sub_event_id = sex.sub_event_id - LEFT JOIN user_driver_profile dpx ON elx.user_driver_id = dpx.driver_id - LEFT JOIN user_codriver_profile cdpx ON elx.user_co_driver_id = cdpx.codriver_id - WHERE (${subClauses.join(' OR ')}) - ) - '''); - } - } - - final whereSql = whereClauses.isNotEmpty ? 'WHERE ${whereClauses.join(' AND ')}' : ''; - final sql = 'SELECT COUNT(DISTINCT ev.event_id) AS count FROM rally_events ev $whereSql;'; - - final result = await query(sql); - if (result.isNotEmpty) { - final countVal = result.first['count']; - if (countVal is int) return countVal; - return int.tryParse(countVal.toString()) ?? 0; - } - return 0; - } - - // =========================================================================== - // 3. SEARCH DRIVER / CO-DRIVER PARTICIPATION RALLIES - // =========================================================================== - - /// Searches rallies persons (drivers and/or co-drivers) participated in via entry_list - Future>> searchDriverRallies(SearchQuery q) async { - final whereClauses = []; - - final personClauses = _buildPersonWhereClauses(q, driverAlias: 'dp', codriverAlias: 'cdp', entryListAlias: 'el'); - if (personClauses.isNotEmpty) { - whereClauses.add("(${personClauses.join(' OR ')})"); - } - - whereClauses.addAll(_buildCountryWhereClauses(q, prefix: 'ev.')); - whereClauses.addAll(_buildCityWhereClauses(q, prefix: 'ev.')); - whereClauses.addAll(_buildYearWhereClauses(q, prefix: 'ev.')); - whereClauses.addAll(_buildRallyWhereClauses(q, prefix: 'ev.')); - - final String personSelectSql; - if (q.driverIds.isNotEmpty || q.driverNames.isNotEmpty) { - final driverMatchClauses = []; - final codriverMatchClauses = []; - if (q.driverIds.isNotEmpty) { - final idsIn = q.driverIds.map((id) => "'${id.replaceAll("'", "''")}'").join(', '); - driverMatchClauses.add("(dp.driver_id IN ($idsIn) OR el.user_driver_id IN ($idsIn))"); - codriverMatchClauses.add("(cdp.codriver_id IN ($idsIn) OR el.user_co_driver_id IN ($idsIn))"); - } - for (final d in q.driverNames) { - final sanitized = d.trim().replaceAll("'", "''").toLowerCase(); - final tokens = sanitized.split(' ').where((t) => t.isNotEmpty).toList(); - if (tokens.length > 1) { - final dTokens = tokens.map((t) => "LOWER(dp.full_name) LIKE '%$t%'").join(' AND '); - final cdTokens = tokens.map((t) => "LOWER(cdp.full_name) LIKE '%$t%'").join(' AND '); - driverMatchClauses.add("($dTokens OR LOWER(dp.nick_name) LIKE '%$sanitized%' OR LOWER(el.driver_link) LIKE '%$sanitized%')"); - codriverMatchClauses.add("($cdTokens OR LOWER(cdp.nick_name) LIKE '%$sanitized%' OR LOWER(el.co_driver_link) LIKE '%$sanitized%')"); - } else { - driverMatchClauses.add("(LOWER(dp.full_name) LIKE '%$sanitized%' OR LOWER(dp.nick_name) LIKE '%$sanitized%' OR LOWER(el.driver_link) LIKE '%$sanitized%')"); - codriverMatchClauses.add("(LOWER(cdp.full_name) LIKE '%$sanitized%' OR LOWER(cdp.nick_name) LIKE '%$sanitized%' OR LOWER(el.co_driver_link) LIKE '%$sanitized%')"); - } - } - final dMatch = "(${driverMatchClauses.join(' OR ')})"; - final cdMatch = "(${codriverMatchClauses.join(' OR ')})"; - - personSelectSql = ''' - CASE - WHEN $cdMatch AND NOT $dMatch THEN cdp.codriver_id - ELSE dp.driver_id - END AS driver_id, - CASE - WHEN $cdMatch AND NOT $dMatch THEN cdp.full_name - ELSE COALESCE(dp.full_name, cdp.full_name, 'Competitor') - END AS driver_name, - CASE - WHEN $dMatch AND $cdMatch THEN 'Driver / Co-Driver' - WHEN $cdMatch THEN 'Co-Driver' - ELSE 'Driver' - END AS role - '''; - } else { - personSelectSql = ''' - COALESCE(dp.driver_id, cdp.codriver_id) AS driver_id, - COALESCE(dp.full_name, cdp.full_name, 'Competitor') AS driver_name, - CASE - WHEN dp.driver_id IS NOT NULL AND cdp.codriver_id IS NOT NULL THEN 'Driver / Co-Driver' - WHEN cdp.codriver_id IS NOT NULL THEN 'Co-Driver' - ELSE 'Driver' - END AS role - '''; - } - - final whereSql = whereClauses.isNotEmpty ? 'WHERE ${whereClauses.join(' AND ')}' : ''; - final sql = ''' - SELECT - ev.event_id AS rally_id, - ev.event_name, - ev.country, - ev.city, - ev.start_date, - $personSelectSql, - MAX(el.car_number) AS car_number, - MAX(COALESCE(el.car, el.make)) AS car, - MAX(el.make) AS make, - NULL AS pos_overall, - NULL AS total_time - FROM rally_entry_list el - JOIN rally_sub_events se ON el.sub_event_id = se.sub_event_id - JOIN rally_events ev ON se.event_id = ev.event_id - LEFT JOIN user_driver_profile dp ON el.user_driver_id = dp.driver_id - LEFT JOIN user_codriver_profile cdp ON el.user_co_driver_id = cdp.codriver_id - $whereSql - GROUP BY ev.event_id, ev.event_name, ev.country, ev.city, ev.start_date, driver_id, driver_name, role - ORDER BY ev.start_date DESC - LIMIT ${q.limit} OFFSET ${q.offset}; - '''; - - return await query(sql); - } - - /// Total count of distinct rally event participations (deduplicating multiple sub-events) - Future countDriverRallies(SearchQuery q) async { - final whereClauses = []; - - final personClauses = _buildPersonWhereClauses(q, driverAlias: 'dp', codriverAlias: 'cdp', entryListAlias: 'el'); - if (personClauses.isNotEmpty) { - whereClauses.add("(${personClauses.join(' OR ')})"); - } - - whereClauses.addAll(_buildCountryWhereClauses(q, prefix: 'ev.')); - whereClauses.addAll(_buildCityWhereClauses(q, prefix: 'ev.')); - whereClauses.addAll(_buildYearWhereClauses(q, prefix: 'ev.')); - whereClauses.addAll(_buildRallyWhereClauses(q, prefix: 'ev.')); - - final whereSql = whereClauses.isNotEmpty ? 'WHERE ${whereClauses.join(' AND ')}' : ''; - final sql = ''' - SELECT COUNT(DISTINCT ev.event_id) AS count - FROM rally_entry_list el - JOIN rally_sub_events se ON el.sub_event_id = se.sub_event_id - JOIN rally_events ev ON se.event_id = ev.event_id - LEFT JOIN user_driver_profile dp ON el.user_driver_id = dp.driver_id - LEFT JOIN user_codriver_profile cdp ON el.user_co_driver_id = cdp.codriver_id - $whereSql; - '''; - - final result = await query(sql); - if (result.isNotEmpty) { - final countVal = result.first['count']; - if (countVal is int) return countVal; - return int.tryParse(countVal.toString()) ?? 0; - } - return 0; - } - - // =========================================================================== - // 4. SEARCH DRIVER WINS - // =========================================================================== - - /// Searches rallies drivers won (1 win counted per rally event on final stage) - Future>> searchDriverWins(SearchQuery q) async { - final whereClauses = [ - 'rr.pos_overall = 1', - _finalStageSubquery, - ]; - - final driverClauses = []; - if (q.driverIds.isNotEmpty) { - final idsIn = q.driverIds.map((id) => "'${id.replaceAll("'", "''")}'").join(', '); - driverClauses.add("(dp.driver_id IN ($idsIn) OR el.user_driver_id IN ($idsIn))"); - } - for (final d in q.driverNames) { - final sanitized = d.trim().replaceAll("'", "''").toLowerCase(); - driverClauses.add("(LOWER(dp.full_name) LIKE '%$sanitized%' OR LOWER(dp.nick_name) LIKE '%$sanitized%' OR LOWER(rr.crew) LIKE '%$sanitized%')"); - } - if (driverClauses.isNotEmpty) { - whereClauses.add("(${driverClauses.join(' OR ')})"); - } - - whereClauses.addAll(_buildCountryWhereClauses(q)); - whereClauses.addAll(_buildCityWhereClauses(q)); - whereClauses.addAll(_buildYearWhereClauses(q)); - whereClauses.addAll(_buildRallyWhereClauses(q)); - - final whereSql = 'WHERE ${whereClauses.join(' AND ')}'; - final sql = ''' - SELECT - ev.event_id AS rally_id, - ev.event_name, - ev.country, - ev.city, - ev.start_date, - dp.driver_id, - COALESCE(dp.full_name, rr.crew) AS driver_name, - rr.crew, - rr.car_number, - el.car, - rr.make, - rr.pos_overall, - rr.total_time - FROM rally_results rr - INNER JOIN rally_events ev ON rr.rally_id = ev.event_id - INNER JOIN rally_stages stg ON rr.stage_id = stg.stage_id - LEFT JOIN rally_entry_list el ON rr.entry_list_id = el.id - LEFT JOIN user_driver_profile dp ON el.user_driver_id = dp.driver_id - $whereSql - GROUP BY rr.id, ev.event_id, ev.event_name, ev.country, ev.city, ev.start_date, dp.driver_id, dp.full_name, rr.crew, rr.car_number, el.car, rr.make, rr.pos_overall, rr.total_time - ORDER BY ev.start_date DESC - LIMIT ${q.limit} OFFSET ${q.offset}; - '''; - - return await query(sql); - } - - /// Count of driver wins - Future countDriverWins(SearchQuery q) async { - final whereClauses = [ - 'rr.pos_overall = 1', - _finalStageSubquery, - ]; - - final driverClauses = []; - if (q.driverIds.isNotEmpty) { - final idsIn = q.driverIds.map((id) => "'${id.replaceAll("'", "''")}'").join(', '); - driverClauses.add("(dp.driver_id IN ($idsIn) OR el.user_driver_id IN ($idsIn))"); - } - for (final d in q.driverNames) { - final sanitized = d.trim().replaceAll("'", "''").toLowerCase(); - driverClauses.add("(LOWER(dp.full_name) LIKE '%$sanitized%' OR LOWER(dp.nick_name) LIKE '%$sanitized%' OR LOWER(rr.crew) LIKE '%$sanitized%')"); - } - if (driverClauses.isNotEmpty) { - whereClauses.add("(${driverClauses.join(' OR ')})"); - } - - whereClauses.addAll(_buildCountryWhereClauses(q)); - whereClauses.addAll(_buildCityWhereClauses(q)); - whereClauses.addAll(_buildYearWhereClauses(q)); - whereClauses.addAll(_buildRallyWhereClauses(q)); - - final whereSql = 'WHERE ${whereClauses.join(' AND ')}'; - final sql = ''' - SELECT COUNT(DISTINCT rr.id) AS count - FROM rally_results rr - INNER JOIN rally_events ev ON rr.rally_id = ev.event_id - INNER JOIN rally_stages stg ON rr.stage_id = stg.stage_id - LEFT JOIN rally_entry_list el ON rr.entry_list_id = el.id - LEFT JOIN user_driver_profile dp ON el.user_driver_id = dp.driver_id - $whereSql; - '''; - - final result = await query(sql); - if (result.isNotEmpty) { - final countVal = result.first['count']; - if (countVal is int) return countVal; - return int.tryParse(countVal.toString()) ?? 0; - } - return 0; - } - - // =========================================================================== - // 5. GET RALLY TOP FINISHERS - // =========================================================================== - - /// Gets the top ranked finishers for rallies on the final classification stage - Future>> getRallyTopFinishers(SearchQuery q) async { - final whereClauses = [ - 'rr.pos_overall IS NOT NULL', - _finalStageSubquery, - ]; - - whereClauses.addAll(_buildRallyWhereClauses(q)); - whereClauses.addAll(_buildYearWhereClauses(q)); - whereClauses.addAll(_buildCountryWhereClauses(q)); - whereClauses.addAll(_buildCityWhereClauses(q)); - - final driverClauses = []; - if (q.driverIds.isNotEmpty) { - final idsIn = q.driverIds.map((id) => "'${id.replaceAll("'", "''")}'").join(', '); - driverClauses.add("(dp.driver_id IN ($idsIn) OR el.user_driver_id IN ($idsIn))"); - } - for (final d in q.driverNames) { - final sanitized = d.trim().replaceAll("'", "''").toLowerCase(); - driverClauses.add("(LOWER(dp.full_name) LIKE '%$sanitized%' OR LOWER(dp.nick_name) LIKE '%$sanitized%' OR LOWER(rr.crew) LIKE '%$sanitized%')"); - } - if (driverClauses.isNotEmpty) { - whereClauses.add("(${driverClauses.join(' OR ')})"); - } - - final whereSql = 'WHERE ${whereClauses.join(' AND ')}'; - final sql = ''' - SELECT - rr.id, - rr.rally_id, - ev.event_name, - rr.stage_id, - stg.stage_name, - stg.stage_number, - dp.driver_id, - COALESCE(dp.full_name, rr.crew) AS driver_name, - rr.crew, - rr.car_number, - rr.make, - rr.class_type, - rr.pos_overall, - rr.pos_stage, - rr.total_time, - rr.stage_time, - rr.diff_leader, - rr.diff_prev - FROM rally_results rr - INNER JOIN rally_events ev ON rr.rally_id = ev.event_id - INNER JOIN rally_stages stg ON rr.stage_id = stg.stage_id - LEFT JOIN rally_entry_list el ON rr.entry_list_id = el.id - LEFT JOIN user_driver_profile dp ON el.user_driver_id = dp.driver_id - $whereSql - GROUP BY rr.id, rr.rally_id, ev.event_name, rr.stage_id, stg.stage_name, stg.stage_number, dp.driver_id, dp.full_name, rr.crew, rr.car_number, rr.make, rr.class_type, rr.pos_overall, rr.pos_stage, rr.total_time, rr.stage_time, rr.diff_leader, rr.diff_prev - ORDER BY rr.pos_overall ASC - LIMIT ${q.limit} OFFSET ${q.offset}; - '''; - - return await query(sql); - } - - /// Count of rally top finishers - Future countRallyTopFinishers(SearchQuery q) async { - final whereClauses = [ - 'rr.pos_overall IS NOT NULL', - _finalStageSubquery, - ]; - - whereClauses.addAll(_buildRallyWhereClauses(q)); - whereClauses.addAll(_buildYearWhereClauses(q)); - whereClauses.addAll(_buildCountryWhereClauses(q)); - whereClauses.addAll(_buildCityWhereClauses(q)); - - final driverClauses = []; - if (q.driverIds.isNotEmpty) { - final idsIn = q.driverIds.map((id) => "'${id.replaceAll("'", "''")}'").join(', '); - driverClauses.add("(dp.driver_id IN ($idsIn) OR el.user_driver_id IN ($idsIn))"); - } - for (final d in q.driverNames) { - final sanitized = d.trim().replaceAll("'", "''").toLowerCase(); - driverClauses.add("(LOWER(dp.full_name) LIKE '%$sanitized%' OR LOWER(dp.nick_name) LIKE '%$sanitized%' OR LOWER(rr.crew) LIKE '%$sanitized%')"); - } - if (driverClauses.isNotEmpty) { - whereClauses.add("(${driverClauses.join(' OR ')})"); - } - - final whereSql = 'WHERE ${whereClauses.join(' AND ')}'; - final sql = ''' - SELECT COUNT(DISTINCT rr.id) AS count - FROM rally_results rr - INNER JOIN rally_events ev ON rr.rally_id = ev.event_id - INNER JOIN rally_stages stg ON rr.stage_id = stg.stage_id - LEFT JOIN rally_entry_list el ON rr.entry_list_id = el.id - LEFT JOIN user_driver_profile dp ON el.user_driver_id = dp.driver_id - $whereSql; - '''; - - final result = await query(sql); - if (result.isNotEmpty) { - final countVal = result.first['count']; - if (countVal is int) return countVal; - return int.tryParse(countVal.toString()) ?? 0; - } - return 0; - } - - // =========================================================================== - // 6. GET RALLY RESULTS (Single Champion Winner) - // =========================================================================== - - /// Gets the first-place winner / single result of a rally - Future>> getRallyResults(SearchQuery q) async { - final singleWinnerQuery = q.copyWith(limit: 1, offset: 0); - final whereClauses = [ - 'rr.pos_overall = 1', - _finalStageSubquery, - ]; - - whereClauses.addAll(_buildRallyWhereClauses(singleWinnerQuery)); - whereClauses.addAll(_buildYearWhereClauses(singleWinnerQuery)); - whereClauses.addAll(_buildCountryWhereClauses(singleWinnerQuery)); - whereClauses.addAll(_buildCityWhereClauses(singleWinnerQuery)); - - final driverClauses = []; - if (singleWinnerQuery.driverIds.isNotEmpty) { - final idsIn = singleWinnerQuery.driverIds.map((id) => "'${id.replaceAll("'", "''")}'").join(', '); - driverClauses.add("(dp.driver_id IN ($idsIn) OR el.user_driver_id IN ($idsIn))"); - } - for (final d in singleWinnerQuery.driverNames) { - final sanitized = d.trim().replaceAll("'", "''").toLowerCase(); - driverClauses.add("(LOWER(dp.full_name) LIKE '%$sanitized%' OR LOWER(dp.nick_name) LIKE '%$sanitized%' OR LOWER(rr.crew) LIKE '%$sanitized%')"); - } - if (driverClauses.isNotEmpty) { - whereClauses.add("(${driverClauses.join(' OR ')})"); - } - - final whereSql = 'WHERE ${whereClauses.join(' AND ')}'; - final sql = ''' - SELECT - rr.id, - rr.rally_id, - ev.event_name, - rr.stage_id, - stg.stage_name, - stg.stage_number, - dp.driver_id, - COALESCE(dp.full_name, rr.crew) AS driver_name, - rr.crew, - rr.car_number, - rr.make, - rr.class_type, - rr.pos_overall, - rr.pos_stage, - rr.total_time, - rr.stage_time, - rr.diff_leader, - rr.diff_prev - FROM rally_results rr - INNER JOIN rally_events ev ON rr.rally_id = ev.event_id - INNER JOIN rally_stages stg ON rr.stage_id = stg.stage_id - LEFT JOIN rally_entry_list el ON rr.entry_list_id = el.id - LEFT JOIN user_driver_profile dp ON el.user_driver_id = dp.driver_id - $whereSql - GROUP BY rr.id, rr.rally_id, ev.event_name, rr.stage_id, stg.stage_name, stg.stage_number, dp.driver_id, dp.full_name, rr.crew, rr.car_number, rr.make, rr.class_type, rr.pos_overall, rr.pos_stage, rr.total_time, rr.stage_time, rr.diff_leader, rr.diff_prev - ORDER BY rr.pos_overall ASC - LIMIT 1; - '''; - - return await query(sql); - } - - // =========================================================================== - // 7. SEARCH DRIVER / CO-DRIVER VIDEOS - // =========================================================================== - - /// Searches videos featuring drivers or co-drivers via metadata -> entry_list -> profile - Future>> searchDriverVideos(SearchQuery q) async { - final whereClauses = [ - "rs.on_demand_url IS NOT NULL AND rs.on_demand_url != ''", - "(rs.video_type IS NULL OR rs.video_type != 'instantReplay')", - ]; - - final personClauses = _buildPersonWhereClauses(q, driverAlias: 'dp', codriverAlias: 'cdp', entryListAlias: 'el'); - if (personClauses.isNotEmpty) { - whereClauses.add("(${personClauses.join(' OR ')})"); - } - - whereClauses.addAll(_buildRallyWhereClauses(q)); - whereClauses.addAll(_buildYearWhereClauses(q)); - whereClauses.addAll(_buildCountryWhereClauses(q)); - whereClauses.addAll(_buildCityWhereClauses(q)); - whereClauses.addAll(_buildStageWhereClauses(q)); - - final whereSql = 'WHERE ${whereClauses.join(' AND ')}'; - final sql = ''' - SELECT - rv.id AS video_id, - MIN(rs.id) AS stream_id, - MIN(rs.on_demand_url) AS on_demand_url, - rv.thumbnail, - ev.event_name, - stg.stage_name, - stg.stage_number, - COALESCE(dp.driver_id, cdp.codriver_id) AS driver_id, - COALESCE(dp.full_name, cdp.full_name) AS driver_name, - rv.video_length_seconds, - rv.created_at - FROM rally_videos rv - INNER JOIN rally_video_metadata vm ON rv.id = vm.video_id - INNER JOIN rally_entry_list el ON vm.entry_list_id = el.id - LEFT JOIN user_driver_profile dp ON el.user_driver_id = dp.driver_id - LEFT JOIN user_codriver_profile cdp ON el.user_co_driver_id = cdp.codriver_id - LEFT JOIN rally_streams rs ON rv.id = rs.video_id - LEFT JOIN rally_stages stg ON rv.stage_id = stg.stage_id - LEFT JOIN rally_events ev ON stg.event_id = ev.event_id - $whereSql - GROUP BY rv.id, rv.thumbnail, ev.event_name, stg.stage_name, stg.stage_number, dp.driver_id, cdp.codriver_id, dp.full_name, cdp.full_name, rv.video_length_seconds, rv.created_at - ORDER BY rv.id DESC - LIMIT ${q.limit} OFFSET ${q.offset}; - '''; - - return await query(sql); - } - - /// Total count of videos featuring drivers or co-drivers - Future countDriverVideos(SearchQuery q) async { - final whereClauses = [ - "rs.on_demand_url IS NOT NULL AND rs.on_demand_url != ''", - "(rs.video_type IS NULL OR rs.video_type != 'instantReplay')", - ]; - - final personClauses = _buildPersonWhereClauses(q, driverAlias: 'dp', codriverAlias: 'cdp', entryListAlias: 'el'); - if (personClauses.isNotEmpty) { - whereClauses.add("(${personClauses.join(' OR ')})"); - } - - whereClauses.addAll(_buildRallyWhereClauses(q)); - whereClauses.addAll(_buildYearWhereClauses(q)); - whereClauses.addAll(_buildCountryWhereClauses(q)); - whereClauses.addAll(_buildCityWhereClauses(q)); - whereClauses.addAll(_buildStageWhereClauses(q)); - - final whereSql = 'WHERE ${whereClauses.join(' AND ')}'; - final sql = ''' - SELECT COUNT(DISTINCT rv.id) AS count - FROM rally_videos rv - INNER JOIN rally_video_metadata vm ON rv.id = vm.video_id - INNER JOIN rally_entry_list el ON vm.entry_list_id = el.id - LEFT JOIN user_driver_profile dp ON el.user_driver_id = dp.driver_id - LEFT JOIN user_codriver_profile cdp ON el.user_co_driver_id = cdp.codriver_id - LEFT JOIN rally_streams rs ON rv.id = rs.video_id - LEFT JOIN rally_stages stg ON rv.stage_id = stg.stage_id - LEFT JOIN rally_events ev ON stg.event_id = ev.event_id - $whereSql; - '''; - - final result = await query(sql); - if (result.isNotEmpty) { - final countVal = result.first['count']; - if (countVal is int) return countVal; - return int.tryParse(countVal.toString()) ?? 0; - } - return 0; - } - - // =========================================================================== - // 8. GET TOP UPLOADERS - // =========================================================================== - - /// Gets top uploaders for a rally or globally, mapped canonically via user_fan_profile -> user_account - Future>> getTopUploaders(SearchQuery q) async { - final whereClauses = ['rv.uploader_user_id IS NOT NULL']; - - whereClauses.addAll(_buildRallyWhereClauses(q)); - whereClauses.addAll(_buildYearWhereClauses(q)); - whereClauses.addAll(_buildCountryWhereClauses(q)); - whereClauses.addAll(_buildCityWhereClauses(q)); - - final whereSql = 'WHERE ${whereClauses.join(' AND ')}'; - final sql = ''' - SELECT - rv.uploader_user_id, - fp.fan_id, - fp.account_id, - COALESCE(NULLIF(TRIM(ua.user_name), ''), NULLIF(TRIM(fp.full_name), ''), NULLIF(TRIM(ua.email), ''), 'Rally Contributor') AS uploader_name, - fp.profile_picture, - COUNT(rv.id) AS upload_count, - MAX(ev.event_name) AS event_name - FROM rally_videos rv - LEFT JOIN user_fan_profile fp ON rv.uploader_user_id = fp.fan_id - LEFT JOIN user_account ua ON fp.account_id = ua.id - LEFT JOIN rally_stages stg ON rv.stage_id = stg.stage_id - LEFT JOIN rally_events ev ON stg.event_id = ev.event_id - $whereSql - GROUP BY rv.uploader_user_id, fp.fan_id, fp.account_id, ua.user_name, fp.full_name, ua.email, fp.profile_picture - ORDER BY upload_count DESC - LIMIT ${q.limit} OFFSET ${q.offset}; - '''; - - return await query(sql); - } - - /// Total count of uploaders - Future countTopUploaders(SearchQuery q) async { - final whereClauses = ['rv.uploader_user_id IS NOT NULL']; - - whereClauses.addAll(_buildRallyWhereClauses(q)); - whereClauses.addAll(_buildYearWhereClauses(q)); - whereClauses.addAll(_buildCountryWhereClauses(q)); - whereClauses.addAll(_buildCityWhereClauses(q)); - - final whereSql = 'WHERE ${whereClauses.join(' AND ')}'; - final sql = ''' - SELECT COUNT(DISTINCT rv.uploader_user_id) AS count - FROM rally_videos rv - LEFT JOIN rally_stages stg ON rv.stage_id = stg.stage_id - LEFT JOIN rally_events ev ON stg.event_id = ev.event_id - $whereSql; - '''; - - final result = await query(sql); - if (result.isNotEmpty) { - final countVal = result.first['count']; - if (countVal is int) return countVal; - return int.tryParse(countVal.toString()) ?? 0; - } - return 0; - } - - // =========================================================================== - // 9. GET TOP DRIVERS BY WINS - // =========================================================================== - - /// Gets ranked leaderboard of drivers with most career rally wins (1 win counted per rally event) - Future>> getTopDriversByWins(SearchQuery q) async { - final whereClauses = [ - 'rr.pos_overall = 1', - _finalStageSubquery, - ]; - - whereClauses.addAll(_buildYearWhereClauses(q)); - whereClauses.addAll(_buildCountryWhereClauses(q)); - - final whereSql = 'WHERE ${whereClauses.join(' AND ')}'; - final sql = ''' - SELECT - dp.driver_id, - COALESCE(dp.full_name, rr.crew) AS driver_name, - dp.country, - dp.profile_picture, - COUNT(DISTINCT rr.rally_id) AS win_count, - MAX(ev.event_name) AS latest_rally_won - FROM rally_results rr - INNER JOIN rally_events ev ON rr.rally_id = ev.event_id - INNER JOIN rally_stages stg ON rr.stage_id = stg.stage_id - LEFT JOIN rally_entry_list el ON rr.entry_list_id = el.id - LEFT JOIN user_driver_profile dp ON el.user_driver_id = dp.driver_id - $whereSql - GROUP BY dp.driver_id, COALESCE(dp.full_name, rr.crew), dp.country, dp.profile_picture - ORDER BY win_count DESC - LIMIT ${q.limit} OFFSET ${q.offset}; - '''; - - return await query(sql); - } - - /// Total count of winning drivers - Future countTopDriversByWins(SearchQuery q) async { - final whereClauses = [ - 'rr.pos_overall = 1', - _finalStageSubquery, - ]; - - whereClauses.addAll(_buildYearWhereClauses(q)); - whereClauses.addAll(_buildCountryWhereClauses(q)); - - final whereSql = 'WHERE ${whereClauses.join(' AND ')}'; - final sql = ''' - SELECT COUNT(DISTINCT COALESCE(dp.driver_id, rr.crew)) AS count - FROM rally_results rr - INNER JOIN rally_events ev ON rr.rally_id = ev.event_id - INNER JOIN rally_stages stg ON rr.stage_id = stg.stage_id - LEFT JOIN rally_entry_list el ON rr.entry_list_id = el.id - LEFT JOIN user_driver_profile dp ON el.user_driver_id = dp.driver_id - $whereSql; - '''; - - final result = await query(sql); - if (result.isNotEmpty) { - final countVal = result.first['count']; - if (countVal is int) return countVal; - return int.tryParse(countVal.toString()) ?? 0; - } - return 0; - } - - /// Normalizes incoming query object (SearchQuery or VideoActionSearchQuery) into SearchQuery - SearchQuery _normalizeSearchQuery(dynamic query) { - if (query is SearchQuery) { - return query; - } - if (query is VideoActionSearchQuery) { - return SearchQuery( - intent: SearchIntent.searchVideoActions, - actionTypes: query.actionType != null ? [query.actionType!] : [], - countries: query.country != null ? [query.country!] : [], - rallyNames: query.eventName != null ? [query.eventName!] : [], - eventNames: query.eventName != null ? [query.eventName!] : [], - stageNames: query.stageName != null ? [query.stageName!] : [], - stageNumbers: query.stageNumber != null ? [query.stageNumber!] : [], - limit: query.limit, - offset: query.offset, - ); - } - return const SearchQuery(intent: SearchIntent.searchRallies); - } - - /// Closes the active database connection - Future close() async { - if (_connection != null && _connection!.connected) { - await _connection!.close(); - _connection = null; - developer.log('MySQL connection closed.', name: 'DatabaseService'); - } - } -} diff --git a/lib/services/entity_search/mysql_entity_search_data_source.dart b/lib/services/entity_search/mysql_entity_search_data_source.dart deleted file mode 100644 index a619b13..0000000 --- a/lib/services/entity_search/mysql_entity_search_data_source.dart +++ /dev/null @@ -1,196 +0,0 @@ -import '../database_service.dart'; -import 'entity_search_models.dart'; -import 'entity_search_service.dart'; - -/// Startup snapshot loader. MySQL remains authoritative; no search SQL or -/// schema detail leaks through [IEntitySearchService]. -class MySqlEntitySearchDataSource implements IEntitySearchDataSource { - final DatabaseService database; - - MySqlEntitySearchDataSource({DatabaseService? database}) - : database = database ?? DatabaseService(); - - @override - Future> loadEntities() async { - // Keep reads sequential: DatabaseService intentionally owns one connection. - final batches = >>[ - await database.query(''' - SELECT event_id, event_name, country, city, - YEAR(start_date) AS event_year - FROM rally_events - WHERE event_name IS NOT NULL AND TRIM(event_name) <> ''; - '''), - await database.query(''' - SELECT account_id, driver_id AS profile_id, full_name, country, - 'driver' AS person_role - FROM user_driver_profile - WHERE driver_id IS NOT NULL AND full_name IS NOT NULL AND TRIM(full_name) <> '' - UNION ALL - SELECT account_id, codriver_id AS profile_id, full_name, country, - 'co_driver' AS person_role - FROM user_codriver_profile - WHERE codriver_id IS NOT NULL AND full_name IS NOT NULL AND TRIM(full_name) <> ''; - '''), - await database.query(''' - SELECT stg.stage_id, stg.stage_name, stg.stage_number, stg.event_id, - ev.event_name - FROM rally_stages stg - LEFT JOIN rally_events ev ON ev.event_id = stg.event_id - WHERE stg.stage_name IS NOT NULL AND TRIM(stg.stage_name) <> ''; - '''), - await database.query(''' - SELECT fp.fan_id, fp.account_id, ua.user_name, fp.full_name, ua.email - FROM user_fan_profile fp - LEFT JOIN user_account ua ON ua.id = fp.account_id - WHERE COALESCE(NULLIF(TRIM(ua.user_name), ''), - NULLIF(TRIM(fp.full_name), ''), - NULLIF(TRIM(ua.email), '')) IS NOT NULL; - '''), - ]; - - final entities = []; - for (final row in batches[0]) { - entities.add( - CanonicalSearchEntity( - canonicalId: row['event_id']?.toString() ?? '', - canonicalName: row['event_name']?.toString() ?? '', - entityType: SearchEntityType.rally, - metadata: { - 'eventId': row['event_id']?.toString(), - 'year': int.tryParse(row['event_year']?.toString() ?? ''), - 'country': row['country']?.toString(), - 'city': row['city']?.toString(), - }, - ), - ); - } - - // One identity per account, even if the account owns both profile types. - final people = >{}; - for (final row in batches[1]) { - final accountId = row['account_id']?.toString() ?? ''; - final profileId = row['profile_id']?.toString() ?? ''; - final profileName = row['full_name']?.toString().trim() ?? ''; - final role = row['person_role']?.toString(); - if (accountId.isEmpty) { - if (profileId.isEmpty || profileName.isEmpty) continue; - final isDriver = role == 'driver'; - entities.add( - CanonicalSearchEntity( - canonicalId: isDriver - ? 'person:driver:$profileId' - : 'person:codriver:$profileId', - canonicalName: profileName, - entityType: SearchEntityType.person, - metadata: { - 'accountId': null, - 'driverId': isDriver ? profileId : null, - 'codriverId': isDriver ? null : profileId, - 'role': isDriver ? 'driver' : 'co_driver', - 'country': row['country']?.toString(), - 'canonicalDisplayName': profileName, - 'searchableNames': [profileName], - 'identityKind': isDriver ? 'driver' : 'codriver', - }, - ), - ); - continue; - } - final current = people.putIfAbsent( - accountId, - () => { - 'country': row['country']?.toString(), - 'role': row['person_role']?.toString(), - 'driverNames': {}, - 'codriverNames': {}, - }, - ); - if (row['person_role'] == 'driver') { - current['driverId'] = row['profile_id']?.toString(); - if (profileName.isNotEmpty) { - (current['driverNames'] as Set).add(profileName); - } - } else { - current['codriverId'] = row['profile_id']?.toString(); - if (profileName.isNotEmpty) { - (current['codriverNames'] as Set).add(profileName); - } - } - if (current['driverId'] != null && current['codriverId'] != null) { - current['role'] = 'both'; - } - } - for (final entry in people.entries) { - final driverNames = (entry.value['driverNames'] as Set).toList() - ..sort((a, b) => a.toLowerCase().compareTo(b.toLowerCase())); - final codriverNames = - (entry.value['codriverNames'] as Set).toList() - ..sort((a, b) => a.toLowerCase().compareTo(b.toLowerCase())); - final searchableNames = { - ...driverNames, - ...codriverNames, - }.toList()..sort((a, b) => a.toLowerCase().compareTo(b.toLowerCase())); - // Deterministic display policy: driver profile name first when present, - // otherwise co-driver profile name; lexical order breaks same-role ties. - final displayName = driverNames.isNotEmpty - ? driverNames.first - : codriverNames.first; - entities.add( - CanonicalSearchEntity( - canonicalId: 'person:account:${entry.key}', - canonicalName: displayName, - entityType: SearchEntityType.person, - metadata: { - 'accountId': entry.key, - 'driverId': entry.value['driverId'], - 'codriverId': entry.value['codriverId'], - 'role': entry.value['role'], - 'country': entry.value['country'], - 'searchableNames': searchableNames, - 'canonicalDisplayName': displayName, - 'identityKind': 'account', - 'canonicalDisplayNamePolicy': - 'driver_profile_then_codriver_profile_lexical', - }, - ), - ); - } - - for (final row in batches[2]) { - entities.add( - CanonicalSearchEntity( - canonicalId: row['stage_id']?.toString() ?? '', - canonicalName: row['stage_name']?.toString() ?? '', - entityType: SearchEntityType.stage, - metadata: { - 'stageId': row['stage_id']?.toString(), - 'stageNumber': row['stage_number']?.toString(), - 'eventId': row['event_id']?.toString(), - 'eventName': row['event_name']?.toString(), - }, - ), - ); - } - for (final row in batches[3]) { - final username = row['user_name']?.toString().trim() ?? ''; - final fullName = row['full_name']?.toString().trim() ?? ''; - final email = row['email']?.toString().trim() ?? ''; - entities.add( - CanonicalSearchEntity( - canonicalId: row['fan_id']?.toString() ?? '', - canonicalName: username.isNotEmpty - ? username - : (fullName.isNotEmpty ? fullName : email), - entityType: SearchEntityType.uploader, - metadata: { - 'fanId': row['fan_id']?.toString(), - 'accountId': row['account_id']?.toString(), - 'username': username, - 'fullName': fullName, - }, - ), - ); - } - return entities; - } -} diff --git a/lib/services/latency/latency_policy.dart b/lib/services/latency/latency_policy.dart new file mode 100644 index 0000000..5c5ec71 --- /dev/null +++ b/lib/services/latency/latency_policy.dart @@ -0,0 +1,76 @@ +/// The single authoritative latency / offline-fallback policy. +/// +/// Every timeout, budget and fallback switch that affects search behaviour is +/// declared here. Nothing else in the app may define a competing budget: the +/// previous code had a 4-second budget inside `OfflineSearchRouter` that the +/// search screen never used, alongside a separate hand-rolled offline path in +/// the screen itself, so the documented behaviour and the shipped behaviour +/// disagreed. One policy object removes that class of drift. +class LatencyPolicy { + /// How long the authoritative online request gets to answer before a valid + /// local result is surfaced in its place. + /// + /// Measured p95 of the warm online path is ~2.3 s, so 4 s leaves real + /// headroom and only trips on genuinely degraded requests. + final Duration onlineResultBudget; + + /// The hard ceiling on one online request. Unchanged from the previous + /// client behaviour (`SearchBackendConfig.typedTimeout`) so this + /// consolidation does not quietly alter when a request is abandoned. + final Duration overallOnlineTimeout; + + /// Voice requests carry an upload and a transcription step, so they keep a + /// longer ceiling. + final Duration overallVoiceTimeout; + + /// When connectivity is known-absent, skip the online attempt entirely + /// rather than burning the budget on a request that cannot succeed. + final bool knownOfflineImmediateFallback; + + /// After a fallback the online request keeps running. It is never cancelled + /// by the fallback itself, and its late result is offered, never applied. + final bool keepOnlineRunningAfterFallback; + + /// A local snapshot older than this is still used, but is labelled as saved + /// data of a stated age. + final Duration snapshotStaleAfter; + + const LatencyPolicy({ + this.onlineResultBudget = const Duration(milliseconds: 4000), + this.overallOnlineTimeout = const Duration(seconds: 35), + this.overallVoiceTimeout = const Duration(seconds: 75), + this.knownOfflineImmediateFallback = true, + this.keepOnlineRunningAfterFallback = true, + this.snapshotStaleAfter = const Duration(hours: 12), + }); + + /// The production policy. Tests construct their own with short budgets. + static const LatencyPolicy standard = LatencyPolicy(); + + LatencyPolicy copyWith({ + Duration? onlineResultBudget, + Duration? overallOnlineTimeout, + Duration? overallVoiceTimeout, + bool? knownOfflineImmediateFallback, + bool? keepOnlineRunningAfterFallback, + Duration? snapshotStaleAfter, + }) { + return LatencyPolicy( + onlineResultBudget: onlineResultBudget ?? this.onlineResultBudget, + overallOnlineTimeout: overallOnlineTimeout ?? this.overallOnlineTimeout, + overallVoiceTimeout: overallVoiceTimeout ?? this.overallVoiceTimeout, + knownOfflineImmediateFallback: + knownOfflineImmediateFallback ?? this.knownOfflineImmediateFallback, + keepOnlineRunningAfterFallback: + keepOnlineRunningAfterFallback ?? this.keepOnlineRunningAfterFallback, + snapshotStaleAfter: snapshotStaleAfter ?? this.snapshotStaleAfter, + ); + } + + @override + String toString() => + 'LatencyPolicy(onlineResultBudgetMs=${onlineResultBudget.inMilliseconds}, ' + 'overallOnlineTimeoutMs=${overallOnlineTimeout.inMilliseconds}, ' + 'knownOfflineImmediateFallback=$knownOfflineImmediateFallback, ' + 'keepOnlineRunningAfterFallback=$keepOnlineRunningAfterFallback)'; +} diff --git a/lib/services/latency/search_latency_coordinator.dart b/lib/services/latency/search_latency_coordinator.dart new file mode 100644 index 0000000..1098cfd --- /dev/null +++ b/lib/services/latency/search_latency_coordinator.dart @@ -0,0 +1,406 @@ +import 'dart:async'; + +import 'package:clock/clock.dart'; + +import '../offline/offline_search_engine.dart'; +import 'latency_policy.dart'; +import 'search_telemetry.dart'; + +/// Reachability signal. Abstracted so the policy is testable without a device. +abstract class ConnectivityProbe { + Future isOnline(); +} + +/// What the coordinator concluded at one point in a search's life. +/// +/// A single search can produce more than one event: a fallback followed later +/// by an offer of the authoritative result. Each event is tagged with the +/// generation it belongs to so a late event from an abandoned search can be +/// discarded by the listener rather than applied. +enum SearchStage { + /// Authoritative online result, within budget. Apply it. + online, + + /// Device known-offline, answered from the local snapshot. Apply it. + offlineImmediate, + + /// Budget elapsed with a valid local result available. Apply it and label it + /// as saved data. The online request is still running. + offlineFallback, + + /// Online failed and a valid local result exists. Apply it and label it. + offlineAfterOnlineFailure, + + /// A late authoritative result arrived after a fallback was shown. + /// DO NOT apply it — offer it. Only a deliberate user action promotes it. + lateOnlineAvailable, + + /// Online failed after a fallback was already shown. Keep the local result + /// on screen; only the labelling changes. + lateOnlineFailed, + + /// Online failed and no safe local answer exists. Surface the error. + onlineFailed, +} + +/// One emission from [SearchLatencyCoordinator.run]. +class SearchEvent { + final int generation; + final SearchStage stage; + + /// Present for [SearchStage.online] and [SearchStage.lateOnlineAvailable]. + final O? online; + + /// Present for the three offline stages. + final OfflineSearchOutcome? offline; + + /// Present for the failure stages. + final Object? error; + + /// Elapsed ms from dispatch to this event. + final int elapsedMs; + + /// Connectivity as observed at dispatch. Reported here so a caller never + /// needs a probe of its own: two probes per search meant two serialized + /// awaits before the online request could even start. + final ConnectivityState connectivity; + + const SearchEvent({ + required this.generation, + required this.stage, + required this.elapsedMs, + this.connectivity = ConnectivityState.unknown, + this.online, + this.offline, + this.error, + }); + + /// Whether this event replaces what is on screen. `lateOnlineAvailable` is + /// deliberately excluded: a late authoritative result is offered, never + /// swapped in behind the user's back. + bool get isTerminalRender => + stage == SearchStage.online || + stage == SearchStage.offlineImmediate || + stage == SearchStage.offlineFallback || + stage == SearchStage.offlineAfterOnlineFailure || + stage == SearchStage.onlineFailed; + + SearchResultSource get source => switch (stage) { + SearchStage.online => SearchResultSource.online, + SearchStage.offlineImmediate => SearchResultSource.offline, + SearchStage.offlineFallback || + SearchStage.offlineAfterOnlineFailure => + SearchResultSource.offlineFallback, + _ => SearchResultSource.online, + }; +} + +/// Progressive online-first search with a bounded local fallback. +/// +/// The shape, in one place: +/// +/// * Known offline -> local immediately, no online attempt. +/// * Online / unknown -> start online now; in parallel prepare the local +/// answer when the query is safely answerable +/// locally; give online [LatencyPolicy.onlineResultBudget]. +/// * Online wins the budget -> show online. +/// * Budget elapses + local -> show local, labelled; the online request keeps +/// running and its late result is *offered*. +/// * Budget elapses, no safe local answer -> keep waiting to the overall +/// timeout. Never fabricate a local result. +/// +/// Online stays authoritative throughout: it is never cancelled by a fallback, +/// and a local result is never silently replaced. +class SearchLatencyCoordinator { + final ConnectivityProbe? connectivity; + final OfflineSearchEngine? engine; + final LatencyPolicy policy; + + const SearchLatencyCoordinator({ + required this.connectivity, + required this.engine, + this.policy = LatencyPolicy.standard, + }); + + /// A local outcome is only allowed to stand in for an authoritative answer + /// when it is unambiguous. + /// + /// A local *clarification* is deliberately not accepted here: showing local + /// disambiguation chips while the authoritative pipeline may be about to + /// answer confidently is exactly the wrong-confidence trade the safety + /// ordering forbids. When the device is plainly offline there is no + /// authoritative alternative, so clarification is accepted there instead — + /// see [_offlineAnswerable]. + static bool _fallbackAnswerable(OfflineSearchOutcome outcome) { + switch (outcome.kind) { + case OfflineOutcomeKind.results: + return (outcome.response?.totalCount ?? 0) > 0; + case OfflineOutcomeKind.special: + return true; + case OfflineOutcomeKind.clarification: + case OfflineOutcomeKind.noMatch: + case OfflineOutcomeKind.unsupported: + return false; + } + } + + /// Whether a plainly-offline device has anything to show. Broader than + /// [_fallbackAnswerable]: with no online path available, an honest + /// clarification or safe no-match is the correct answer. + static bool _offlineAnswerable(OfflineSearchOutcome outcome) => + outcome.kind != OfflineOutcomeKind.unsupported; + + /// Runs one search. [generation] identifies this dispatch; every event + /// carries it back so the caller can drop results from superseded searches. + /// + /// The returned stream closes once no further events are possible. Cancelling + /// the subscription stops delivery; it does not cancel the online request, + /// which is allowed to complete (and be ignored) rather than leaving the + /// backend with a half-abandoned call. + Stream> run({ + required int generation, + required String rawText, + required Future Function() online, + int limit = 20, + int offset = 0, + }) { + final controller = StreamController>(); + final startedAt = clock.now(); + // Completes when the listener detaches (the screen was disposed, or a newer + // query superseded this one). Everything the coordinator still has running + // stops at the next await, so no timer outlives the caller. + final cancelled = Completer(); + var closed = false; + var observed = ConnectivityState.unknown; + + void emit( + SearchStage stage, { + O? online, + OfflineSearchOutcome? offline, + Object? error, + }) { + if (closed || controller.isClosed) return; + controller.add(SearchEvent( + generation: generation, + stage: stage, + elapsedMs: clock.now().difference(startedAt).inMilliseconds, + connectivity: observed, + online: online, + offline: offline, + error: error, + )); + } + + void finish() { + if (closed) return; + closed = true; + // Closing from inside an `add` is safe, but scheduling it keeps the + // final event and the done signal in a predictable order for listeners. + scheduleMicrotask(() { + if (!controller.isClosed) controller.close(); + }); + } + + controller.onCancel = () { + if (!cancelled.isCompleted) cancelled.complete(); + }; + controller.onListen = () { + unawaited(_drive( + rawText: rawText, + online: online, + limit: limit, + offset: offset, + emit: emit, + finish: finish, + setConnectivity: (state) => observed = state, + cancelled: cancelled.future, + isCancelled: () => cancelled.isCompleted, + )); + }; + return controller.stream; + } + + Future _drive({ + required String rawText, + required Future Function() online, + required int limit, + required int offset, + required void Function(SearchStage, + {O? online, OfflineSearchOutcome? offline, Object? error}) + emit, + required void Function() finish, + required void Function(ConnectivityState) setConnectivity, + required Future cancelled, + required bool Function() isCancelled, + }) async { + final localEngine = engine; + + /// Waits for the online request, the overall timeout, or cancellation — + /// whichever comes first — leaving no timer behind in any of the three + /// cases. `Future.timeout` would keep its timer alive for the full budget + /// even after the caller has gone away. + Future awaitOnlineOrTimeout(Future online) async { + final expired = Completer(); + final timer = Timer(policy.overallOnlineTimeout, () { + if (!expired.isCompleted) expired.complete(); + }); + try { + await Future.any([online, expired.future, cancelled]); + } finally { + timer.cancel(); + } + return !expired.isCompleted; + } + + Future runLocal() async { + if (localEngine == null || rawText.trim().isEmpty) return null; + try { + if (!await localEngine.database.hasSnapshot()) return null; + return await localEngine.search(rawText, limit: limit, offset: offset); + } catch (_) { + // A missing, corrupt or half-synced snapshot must never take down the + // search: the online path stays authoritative and simply has no + // fallback to offer. + return null; + } + } + + // The single connectivity read for this search. + final connectivity = await observedConnectivity(); + setConnectivity(connectivity); + + // A. Known offline -> answer locally now, do not wait on a request that + // cannot succeed. + if (policy.knownOfflineImmediateFallback && + connectivity == ConnectivityState.offline) { + final outcome = await runLocal(); + if (outcome != null && _offlineAnswerable(outcome)) { + emit(SearchStage.offlineImmediate, offline: outcome); + } else { + emit(SearchStage.onlineFailed, + error: StateError('offline with no usable local answer')); + } + finish(); + return; + } + + // B. Online or unknown connectivity: start the authoritative request + // immediately, and prepare the local answer alongside it. + final onlineFuture = online(); + // Attach a no-op handler now so a failure that lands after the budget can + // never surface as an unhandled async error. + var onlineSettled = false; + Object? onlineError; + O? onlineValue; + final guardedOnline = onlineFuture.then((value) { + onlineSettled = true; + onlineValue = value; + }, onError: (Object error, StackTrace _) { + onlineSettled = true; + onlineError = error; + }); + + final localFuture = runLocal(); + + final budgetElapsed = Completer(); + final budgetTimer = Timer(policy.onlineResultBudget, () { + if (!budgetElapsed.isCompleted) budgetElapsed.complete(); + }); + + // Race the online request against the budget. + await Future.any([guardedOnline, budgetElapsed.future, cancelled]); + budgetTimer.cancel(); + if (isCancelled()) { + finish(); + return; + } + + if (onlineSettled && onlineError == null) { + // Online answered inside the budget. It wins even if the local result + // was ready first — that is the deterministic rule for the + // near-simultaneous case. + emit(SearchStage.online, online: onlineValue); + finish(); + return; + } + + if (onlineSettled && onlineError != null) { + // Online failed before the budget elapsed. + final outcome = await localFuture; + if (outcome != null && _fallbackAnswerable(outcome)) { + emit(SearchStage.offlineAfterOnlineFailure, offline: outcome); + } else { + emit(SearchStage.onlineFailed, error: onlineError); + } + finish(); + return; + } + + // Budget elapsed with the online request still in flight. + final outcome = await localFuture; + final canFallBack = outcome != null && _fallbackAnswerable(outcome); + + if (!canFallBack) { + // E. No safe local answer: keep waiting for the authoritative result + // under the normal overall timeout. Nothing is fabricated. + final answered = await awaitOnlineOrTimeout(guardedOnline); + if (isCancelled()) { + finish(); + return; + } + if (!answered) { + emit( + SearchStage.onlineFailed, + error: TimeoutException( + 'online search exceeded the overall timeout', + policy.overallOnlineTimeout, + ), + ); + } else if (onlineError != null) { + emit(SearchStage.onlineFailed, error: onlineError); + } else { + emit(SearchStage.online, online: onlineValue); + } + finish(); + return; + } + + // C. Show the local result now, keep the online request running. + emit(SearchStage.offlineFallback, offline: outcome); + + if (!policy.keepOnlineRunningAfterFallback) { + finish(); + return; + } + + final answered = await awaitOnlineOrTimeout(guardedOnline); + if (isCancelled()) { + finish(); + return; + } + if (!answered || onlineError != null) { + // D. Online failed (or never arrived) after the fallback: the local + // result stays exactly as it is. + emit(SearchStage.lateOnlineFailed, error: onlineError); + } else { + // The authoritative result is ready but is only ever *offered*. + emit(SearchStage.lateOnlineAvailable, online: onlineValue); + } + finish(); + } + + /// Reads the probe once. A probe that is absent or throwing yields + /// [ConnectivityState.unknown], which is treated as "attempt online" — never + /// as offline, so an unusable probe cannot suppress the authoritative path. + Future observedConnectivity() async { + final probe = connectivity; + if (probe == null) return ConnectivityState.unknown; + try { + return await probe.isOnline() + ? ConnectivityState.online + : ConnectivityState.offline; + } catch (_) { + return ConnectivityState.unknown; + } + } +} diff --git a/lib/services/latency/search_telemetry.dart b/lib/services/latency/search_telemetry.dart new file mode 100644 index 0000000..0b6f3b2 --- /dev/null +++ b/lib/services/latency/search_telemetry.dart @@ -0,0 +1,121 @@ +import 'dart:math'; + +/// Where the result currently on screen came from. +enum SearchResultSource { + /// Authoritative result from the backend. + online, + + /// Device was known-offline; the local snapshot answered. + offline, + + /// Online was tried, exceeded its budget (or failed), and the local snapshot + /// answered in its place. The online request is still running or has failed. + offlineFallback, +} + +extension SearchResultSourceWire on SearchResultSource { + String get wireName => switch (this) { + SearchResultSource.online => 'online', + SearchResultSource.offline => 'offline', + SearchResultSource.offlineFallback => 'offline_fallback', + }; +} + +/// Client-side connectivity as understood at dispatch time. +enum ConnectivityState { online, offline, unknown } + +/// One search's client-side latency record. +/// +/// Deliberately carries no query text, no result payload and no credentials — +/// only the correlation id, durations and small enum-like flags. The +/// `requestId` matches the `X-Request-Id` sent to the backend, so a client +/// record and a backend timing line can be joined without either side logging +/// user content. +class SearchTelemetry { + final String requestId; + final int totalClientMs; + + /// Wall time of the online HTTP call, when one was actually issued. + final int? networkRoundtripMs; + + /// Time from dispatch to the first result rendered, whatever its source. + final int? timeToFirstResultMs; + + final bool fallbackTriggered; + + /// Elapsed ms at which the fallback fired. Null when it did not. + final int? fallbackTriggerMs; + + final SearchResultSource resultSource; + final ConnectivityState connectivity; + + /// Whether the deterministic local parser produced a result it could stand + /// behind for this query. False for ambiguous, unsupported or no-match + /// queries, which must never be forced into a local answer. + final bool localParserCouldAnswer; + + /// Set when a late online result arrived after a fallback and is waiting for + /// the user to accept it. + final bool lateOnlineOffered; + + const SearchTelemetry({ + required this.requestId, + required this.totalClientMs, + required this.resultSource, + required this.connectivity, + required this.fallbackTriggered, + required this.localParserCouldAnswer, + this.networkRoundtripMs, + this.timeToFirstResultMs, + this.fallbackTriggerMs, + this.lateOnlineOffered = false, + }); + + Map toJson() => { + 'request_id': requestId, + 'total_client_ms': totalClientMs, + if (networkRoundtripMs != null) 'network_roundtrip_ms': networkRoundtripMs, + if (timeToFirstResultMs != null) 'time_to_first_result_ms': timeToFirstResultMs, + 'fallback_triggered': fallbackTriggered, + if (fallbackTriggerMs != null) 'fallback_trigger_ms': fallbackTriggerMs, + 'result_source': resultSource.wireName, + 'connectivity': connectivity.name, + 'local_parser_could_answer': localParserCouldAnswer, + 'late_online_offered': lateOnlineOffered, + }; +} + +/// Generates the correlation id sent as `X-Request-Id`. +/// +/// Opaque and derived only from time plus randomness — it carries nothing +/// about the user or the query. +String newRequestId() { + const alphabet = '0123456789abcdef'; + final random = Random(); + final buffer = StringBuffer(); + for (var i = 0; i < 32; i++) { + buffer.write(alphabet[random.nextInt(16)]); + } + return buffer.toString(); +} + +/// Receives completed search telemetry. Production wires this to structured +/// logging; tests capture the records directly. +abstract class SearchTelemetrySink { + void record(SearchTelemetry telemetry); +} + +/// Discards everything. The default, so instrumentation costs nothing unless a +/// sink is deliberately installed. +class NullSearchTelemetrySink implements SearchTelemetrySink { + const NullSearchTelemetrySink(); + @override + void record(SearchTelemetry telemetry) {} +} + +/// Keeps records in memory for tests and local debugging. +class InMemorySearchTelemetrySink implements SearchTelemetrySink { + final List records = []; + @override + void record(SearchTelemetry telemetry) => records.add(telemetry); +} diff --git a/lib/services/llm/entity_resolution/entity_lookup_repository.dart b/lib/services/llm/entity_resolution/entity_lookup_repository.dart index 13d2f7f..80002c9 100644 --- a/lib/services/llm/entity_resolution/entity_lookup_repository.dart +++ b/lib/services/llm/entity_resolution/entity_lookup_repository.dart @@ -1,10 +1,11 @@ import '../../../models/entity_candidate.dart'; import '../../../models/search_query.dart'; -import '../../database_service.dart'; -import 'phonetic_matching_helper.dart'; -import 'transliteration_helper.dart'; -/// Abstract contract for database-backed entity candidate lookups. +/// Abstract contract for entity candidate lookups. +/// +/// Implementations are backend-backed (over HTTPS) or test doubles. The device +/// never queries the database directly; the former AWS RDS/MySQL implementation +/// has been removed so no DB credentials or SQL ship in the client. abstract class IEntityLookupRepository { Future> lookupRallies( String phrase, { @@ -41,823 +42,3 @@ abstract class IEntityLookupRepository { int limit = 25, }); } - -/// Production implementation of IEntityLookupRepository using AWS RDS MySQL via DatabaseService. -class DatabaseEntityLookupRepository implements IEntityLookupRepository { - final DatabaseService _dbService; - - DatabaseEntityLookupRepository({DatabaseService? dbService}) - : _dbService = dbService ?? DatabaseService(); - - /// Generates a bounded, high-recall list of candidate search patterns incorporating - /// normalized full phrase, space-collapsed, descriptor-stripped stem, 3-char token prefixes, - /// and end-anchors for long tokens. - List _buildCandidatePatterns(String phrase) { - final clean = phrase.trim().replaceAll("'", "''"); - if (clean.isEmpty) return []; - - final patterns = {}; - final normalized = PhoneticMatchingHelper.normalize(clean); - if (normalized.isNotEmpty) patterns.add(normalized); - - final collapsed = PhoneticMatchingHelper.collapseSpaces(clean); - if (collapsed.isNotEmpty) patterns.add(collapsed); - - final coreStem = PhoneticMatchingHelper.stripDescriptors(normalized); - if (coreStem.isNotEmpty && coreStem != normalized) { - patterns.add(coreStem); - final collapsedCore = PhoneticMatchingHelper.collapseSpaces(coreStem); - if (collapsedCore.isNotEmpty) patterns.add(collapsedCore); - } - - // Conservative acoustic-folded comparison representations - final acousticNorm = PhoneticMatchingHelper.acousticFold(normalized); - if (acousticNorm.isNotEmpty && acousticNorm != normalized) { - patterns.add(acousticNorm); - } - final acousticCollapsed = PhoneticMatchingHelper.acousticFold(collapsed); - if (acousticCollapsed.isNotEmpty && acousticCollapsed != collapsed) { - patterns.add(acousticCollapsed); - } - - // Cross-script transliterations - if (TransliterationHelper.isArabicOrUrdu(clean)) { - final translits = TransliterationHelper.transliterateToLatin(clean); - for (final t in translits) { - final tNorm = PhoneticMatchingHelper.normalize(t); - if (tNorm.isNotEmpty) patterns.add(tNorm); - final tCollapsed = PhoneticMatchingHelper.collapseSpaces(t); - if (tCollapsed.isNotEmpty) patterns.add(tCollapsed); - } - } - - // Token prefixes (length >= 3) and anchor fragments - // Filter out generic motorsport words and years from standalone single-token patterns - const genericWords = { - 'rally', - 'rallye', - 'rali', - 'rajd', - 'rallijsprints', - 'stage', - 'stages', - 'forestry', - 'championship', - 'series', - 'international', - 'regional', - }; - - final tokens = normalized.split(' ').where((t) => t.length >= 3).toList(); - if (tokens.length == 2) { - patterns.add('${tokens[1]} ${tokens[0]}'); - } - final distinctiveTokens = tokens - .where( - (t) => !genericWords.contains(t) && !RegExp(r'^\d{4}$').hasMatch(t), - ) - .toList(); - final targetTokens = distinctiveTokens.isNotEmpty - ? distinctiveTokens - : tokens; - - // Prioritize distinctive tokens - final orderedTokens = targetTokens.length > 1 - ? [ - targetTokens.last, - ...targetTokens.sublist(0, targetTokens.length - 1), - ] - : targetTokens; - - for (final token in orderedTokens) { - patterns.add(token); - - // Irish/Scottish O' / Mc / Mac prefix decomposition (e.g. oconnor -> connor, mcrae -> rae) - if (token.startsWith('o') && token.length >= 5) { - patterns.add(token.substring(1)); - patterns.add("o'${token.substring(1)}"); - patterns.add("o ${token.substring(1)}"); - } else if (token.startsWith('mc') && token.length >= 5) { - patterns.add(token.substring(2)); - } else if (token.startsWith('mac') && token.length >= 6) { - patterns.add(token.substring(3)); - } - - // 3-character & 4-character prefix for phonetic/orthographic tolerance - if (token.length >= 3) { - patterns.add(token.substring(0, 3)); - } - if (token.length >= 4) { - patterns.add(token.substring(0, 4)); - } - - // Generalized root stem for long words by trimming common phonetic/transcription tail (e.g. loncarich -> loncar, bogovich -> bogov) - if (token.length >= 6) { - patterns.add(token.substring(0, token.length - 2)); - } - if (token.length >= 7) { - patterns.add(token.substring(0, token.length - 3)); - } - - // Suffix/anchor fragment for long words (length >= 6) to catch prefix acoustic errors - if (token.length >= 6) { - final mid = token.substring(3); - if (mid.length >= 3) { - patterns.add(mid); - } - } - } - - // Bounded internal character n-grams from collapsed and acoustic representations - final ngrams = PhoneticMatchingHelper.generateNgramAnchors( - collapsed, - n: 3, - maxAnchors: 4, - ); - patterns.addAll(ngrams); - if (acousticCollapsed != collapsed) { - final acNgrams = PhoneticMatchingHelper.generateNgramAnchors( - acousticCollapsed, - n: 3, - maxAnchors: 3, - ); - patterns.addAll(acNgrams); - } - - // Bounded budget: Return at most 12 most informative unique patterns - return patterns.take(12).toList(); - } - - @override - Future> lookupRallies( - String phrase, { - int? year, - String? country, - String? city, - int limit = 35, - }) async { - final clean = phrase.trim().replaceAll("'", "''"); - if (clean.isEmpty) return []; - - final patterns = _buildCandidatePatterns(phrase); - final patternClauses = []; - for (final p in patterns) { - final pEscaped = p.replaceAll("'", "''").toLowerCase(); - patternClauses.add("LOWER(ev.event_name) LIKE '%$pEscaped%'"); - if (pEscaped.length >= 3 && !pEscaped.contains(' ')) { - patternClauses.add( - "REPLACE(LOWER(ev.event_name), ' ', '') LIKE '%$pEscaped%'", - ); - } - } - - patternClauses.add("ev.event_id = '$clean'"); - - final whereClauses = ['(${patternClauses.join(' OR ')})']; - - if (year != null && year > 0) { - whereClauses.add( - "(YEAR(ev.start_date) = $year OR YEAR(ev.end_date) = $year)", - ); - } - - if (country != null && - country.trim().isNotEmpty && - country.toUpperCase() != 'ALL') { - final sanitizedCountry = country - .trim() - .replaceAll("'", "''") - .toLowerCase(); - whereClauses.add("LOWER(ev.country) LIKE '%$sanitizedCountry%'"); - } - - if (city != null && city.trim().isNotEmpty && city.toUpperCase() != 'ALL') { - final sanitizedCity = city.trim().replaceAll("'", "''").toLowerCase(); - whereClauses.add("LOWER(ev.city) LIKE '%$sanitizedCity%'"); - } - - final normalized = PhoneticMatchingHelper.normalize(clean); - final coreStem = PhoneticMatchingHelper.stripDescriptors(normalized); - final distinctive = coreStem.isNotEmpty ? coreStem : normalized; - final distEscaped = distinctive.replaceAll("'", "''").toLowerCase(); - - final whereSql = 'WHERE ${whereClauses.join(' AND ')}'; - final sql = - ''' - SELECT - ev.event_id, - ev.event_name, - ev.country, - ev.city, - YEAR(ev.start_date) AS event_year, - ev.start_date - FROM rally_events ev - $whereSql - ORDER BY - CASE - WHEN LOWER(ev.event_name) = '${clean.toLowerCase()}' THEN 1 - WHEN LOWER(ev.event_name) LIKE '${clean.toLowerCase()}%' THEN 2 - WHEN LOWER(ev.event_name) LIKE '%$distEscaped%' THEN 3 - ELSE 4 - END, - ev.start_date DESC - LIMIT $limit; - '''; - - final rows = await _dbService.query(sql); - return rows.map((r) { - final eventId = r['event_id']?.toString() ?? ''; - final name = r['event_name']?.toString() ?? ''; - final cCountry = r['country']?.toString(); - final cCity = r['city']?.toString(); - final cYear = r['event_year']?.toString(); - - final parts = []; - if (cCountry != null && cCountry.isNotEmpty) parts.add(cCountry); - if (cCity != null && cCity.isNotEmpty) parts.add(cCity); - if (cYear != null && cYear.isNotEmpty) parts.add(cYear); - - return EntityCandidate( - id: eventId, - type: EntityType.rally, - canonicalName: name, - subtitle: parts.isNotEmpty ? parts.join(' • ') : null, - metadata: { - 'country': cCountry, - 'city': cCity, - 'year': int.tryParse(cYear ?? ''), - }, - ); - }).toList(); - } - - @override - Future> lookupDrivers( - String phrase, { - String? eventId, - String? eventName, - int? year, - PersonRole personRole = PersonRole.any, - int limit = 35, - }) async { - final clean = phrase.trim().replaceAll("'", "''"); - if (clean.isEmpty) return []; - - final patterns = _buildCandidatePatterns(phrase); - final driverPatternClauses = []; - final codriverPatternClauses = []; - for (final p in patterns) { - final pEscaped = p.replaceAll("'", "''").toLowerCase(); - driverPatternClauses.add("LOWER(dp.full_name) LIKE '%$pEscaped%'"); - codriverPatternClauses.add("LOWER(cdp.full_name) LIKE '%$pEscaped%'"); - if (pEscaped.length >= 3 && !pEscaped.contains(' ')) { - driverPatternClauses.add( - "REPLACE(LOWER(dp.full_name), ' ', '') LIKE '%$pEscaped%'", - ); - codriverPatternClauses.add( - "REPLACE(LOWER(cdp.full_name), ' ', '') LIKE '%$pEscaped%'", - ); - } - } - - driverPatternClauses.add( - "LOWER(dp.nick_name) LIKE '%${clean.toLowerCase()}%'", - ); - codriverPatternClauses.add( - "LOWER(cdp.nick_name) LIKE '%${clean.toLowerCase()}%'", - ); - - final driverNameMatchSql = '(${driverPatternClauses.join(' OR ')})'; - final codriverNameMatchSql = '(${codriverPatternClauses.join(' OR ')})'; - - final normalized = PhoneticMatchingHelper.normalize(clean); - final tokens = normalized.split(' ').where((t) => t.length >= 3).toList(); - final surnameToken = tokens.isNotEmpty - ? tokens.last.replaceAll("'", "''") - : clean.replaceAll("'", "''"); - final surnamePrefix = surnameToken.length >= 3 - ? surnameToken.substring(0, 3) - : surnameToken; - - final driverRankSql = - ''' - CASE - WHEN LOWER(dp.full_name) = '${clean.toLowerCase()}' THEN 1 - WHEN LOWER(dp.full_name) LIKE '${clean.toLowerCase()}%' THEN 2 - WHEN LOWER(dp.full_name) LIKE '%$surnameToken%' THEN 3 - WHEN LOWER(dp.full_name) LIKE '%$surnamePrefix%' THEN 4 - ELSE 5 - END - '''; - - final codriverRankSql = - ''' - CASE - WHEN LOWER(cdp.full_name) = '${clean.toLowerCase()}' THEN 1 - WHEN LOWER(cdp.full_name) LIKE '${clean.toLowerCase()}%' THEN 2 - WHEN LOWER(cdp.full_name) LIKE '%$surnameToken%' THEN 3 - WHEN LOWER(cdp.full_name) LIKE '%$surnamePrefix%' THEN 4 - ELSE 5 - END - '''; - - // If context (event or year) is provided, prioritize participants - if (eventId != null || eventName != null || (year != null && year > 0)) { - final driverContextClauses = [driverNameMatchSql]; - final codriverContextClauses = [codriverNameMatchSql]; - if (eventId != null && eventId.isNotEmpty) { - final sanitizedEvId = eventId.replaceAll("'", "''"); - driverContextClauses.add("ev.event_id = '$sanitizedEvId'"); - codriverContextClauses.add("ev.event_id = '$sanitizedEvId'"); - } else if (eventName != null && eventName.isNotEmpty) { - final sanitizedEv = eventName.replaceAll("'", "''").toLowerCase(); - driverContextClauses.add("LOWER(ev.event_name) LIKE '%$sanitizedEv%'"); - codriverContextClauses.add( - "LOWER(ev.event_name) LIKE '%$sanitizedEv%'", - ); - } - if (year != null && year > 0) { - driverContextClauses.add( - "(YEAR(ev.start_date) = $year OR YEAR(ev.end_date) = $year)", - ); - codriverContextClauses.add( - "(YEAR(ev.start_date) = $year OR YEAR(ev.end_date) = $year)", - ); - } - - final contextSql = - ''' - (SELECT DISTINCT - dp.driver_id AS id, - dp.account_id, - dp.full_name, - dp.nick_name, - dp.country, - 'driver' AS role, - MAX(ev.event_name) AS participated_event, - MAX(YEAR(ev.start_date)) AS event_year, - $driverRankSql AS match_rank - FROM user_driver_profile dp - INNER JOIN rally_entry_list el ON el.user_driver_id = dp.driver_id - INNER JOIN rally_sub_events se ON el.sub_event_id = se.sub_event_id - INNER JOIN rally_events ev ON se.event_id = ev.event_id - WHERE ${driverContextClauses.join(' AND ')} - GROUP BY dp.driver_id, dp.account_id, dp.full_name, dp.nick_name, dp.country) - UNION ALL - (SELECT DISTINCT - cdp.codriver_id AS id, - cdp.account_id, - cdp.full_name, - cdp.nick_name, - cdp.country, - 'co_driver' AS role, - MAX(ev.event_name) AS participated_event, - MAX(YEAR(ev.start_date)) AS event_year, - $codriverRankSql AS match_rank - FROM user_codriver_profile cdp - INNER JOIN rally_entry_list el ON el.user_co_driver_id = cdp.codriver_id - INNER JOIN rally_sub_events se ON el.sub_event_id = se.sub_event_id - INNER JOIN rally_events ev ON se.event_id = ev.event_id - WHERE ${codriverContextClauses.join(' AND ')} - GROUP BY cdp.codriver_id, cdp.account_id, cdp.full_name, cdp.nick_name, cdp.country) - ORDER BY match_rank ASC - LIMIT 60; - '''; - - final contextRows = await _dbService.query(contextSql); - if (contextRows.isNotEmpty) { - return await _mergeAndMapPersonCandidates( - contextRows, - inContext: true, - cleanPhrase: clean, - limit: limit, - ); - } - } - - // General lookup across BOTH user_driver_profile and user_codriver_profile - final sql = - ''' - (SELECT - dp.driver_id AS id, - dp.account_id, - dp.full_name, - dp.nick_name, - dp.country, - 'driver' AS role, - NULL AS participated_event, - NULL AS event_year, - $driverRankSql AS match_rank - FROM user_driver_profile dp - WHERE $driverNameMatchSql - ORDER BY match_rank ASC - LIMIT 50) - UNION ALL - (SELECT - cdp.codriver_id AS id, - cdp.account_id, - cdp.full_name, - cdp.nick_name, - cdp.country, - 'co_driver' AS role, - NULL AS participated_event, - NULL AS event_year, - $codriverRankSql AS match_rank - FROM user_codriver_profile cdp - WHERE $codriverNameMatchSql - ORDER BY match_rank ASC - LIMIT 50) - ORDER BY match_rank ASC; - '''; - - final rows = await _dbService.query(sql); - return await _mergeAndMapPersonCandidates( - rows, - inContext: false, - cleanPhrase: clean, - limit: limit, - ); - } - - /// Consolidates person rows from driver and co-driver tables into unified candidates. - /// Uses account_id as the authoritative cross-role identity bridge. - Future> _mergeAndMapPersonCandidates( - List> rows, { - required bool inContext, - required String cleanPhrase, - required int limit, - }) async { - // Collect all matched account_ids to discover cross-role profiles - final matchedAccountIds = {}; - for (final r in rows) { - final acc = r['account_id']?.toString()?.trim(); - if (acc != null && acc.isNotEmpty && acc != 'null') { - matchedAccountIds.add(acc); - } - } - - List> allRows = List>.from(rows); - - // If account_ids exist, query both user_driver_profile and user_codriver_profile for complete cross-role discovery - if (matchedAccountIds.isNotEmpty) { - final accIn = matchedAccountIds - .map((a) => "'${a.replaceAll("'", "''")}'") - .join(', '); - final crossRoleSql = - ''' - (SELECT - dp.driver_id AS id, - dp.account_id, - dp.full_name, - dp.nick_name, - dp.country, - 'driver' AS role, - NULL AS participated_event, - NULL AS event_year, - 1 AS match_rank - FROM user_driver_profile dp - WHERE dp.account_id IN ($accIn)) - UNION ALL - (SELECT - cdp.codriver_id AS id, - cdp.account_id, - cdp.full_name, - cdp.nick_name, - cdp.country, - 'co_driver' AS role, - NULL AS participated_event, - NULL AS event_year, - 1 AS match_rank - FROM user_codriver_profile cdp - WHERE cdp.account_id IN ($accIn)); - '''; - final crossRows = await _dbService.query(crossRoleSql); - if (crossRows.isNotEmpty) { - allRows.addAll(crossRows); - } - } - - // Group by account_id (authoritative) or normalized full_name fallback - final merged = >{}; - - for (final r in allRows) { - final name = r['full_name']?.toString()?.trim() ?? ''; - if (name.isEmpty) continue; - final accountId = r['account_id']?.toString()?.trim(); - final hasAcc = - accountId != null && accountId.isNotEmpty && accountId != 'null'; - final key = hasAcc ? 'acc:$accountId' : 'name:${name.toLowerCase()}'; - - if (!merged.containsKey(key)) { - final entry = Map.from(r); - if (r['role'] == 'driver') { - entry['driver_id'] = r['id']; - } else if (r['role'] == 'co_driver') { - entry['codriver_id'] = r['id']; - } - merged[key] = entry; - } else { - final existing = merged[key]!; - final existingRole = existing['role']?.toString(); - final currentRole = r['role']?.toString(); - if (existingRole != null && - currentRole != null && - existingRole != currentRole) { - existing['role'] = 'both'; - } - if (currentRole == 'driver') { - existing['driver_id'] = r['id']; - // Preserve driver name if existing was empty - if (existing['driver_name'] == null) - existing['driver_name'] = r['full_name']; - } else if (currentRole == 'co_driver') { - existing['codriver_id'] = r['id']; - if (existing['codriver_name'] == null) - existing['codriver_name'] = r['full_name']; - } - if (r['participated_event'] != null) { - existing['participated_event'] = r['participated_event']; - } - if (r['event_year'] != null) { - existing['event_year'] = r['event_year']; - } - } - } - - final candidates = merged.values.map((r) { - final role = r['role']?.toString() ?? 'driver'; - final accountId = r['account_id']?.toString()?.trim(); - final driverId = - r['driver_id']?.toString() ?? - (role == 'driver' ? r['id']?.toString() : null); - final codriverId = - r['codriver_id']?.toString() ?? - (role == 'co_driver' ? r['id']?.toString() : null); - final id = driverId ?? codriverId ?? r['id']?.toString() ?? ''; - final name = r['full_name']?.toString() ?? ''; - final country = r['country']?.toString(); - final event = r['participated_event']?.toString(); - final yr = r['event_year']?.toString(); - - final parts = []; - if (role == 'both') { - parts.add('DRIVER / CO-DRIVER'); - } else if (role == 'co_driver') { - parts.add('CO-DRIVER'); - } else { - parts.add('DRIVER'); - } - if (country != null && country.isNotEmpty) - parts.add(country.toUpperCase()); - if (event != null && event.isNotEmpty) parts.add(event); - if (yr != null && yr.isNotEmpty) parts.add(yr); - - return EntityCandidate( - id: id, - type: EntityType.driver, - canonicalName: name, - subtitle: parts.isNotEmpty ? parts.join(' • ') : null, - metadata: { - 'country': country, - 'role': role, - 'accountId': accountId, - 'driverId': driverId, - 'codriverId': codriverId, - 'inContext': inContext, - 'year': int.tryParse(yr ?? ''), - 'matchRank': r['match_rank'], - }, - ); - }).toList(); - - // Sort by match rank first, then exact matches - candidates.sort((a, b) { - final aRank = - int.tryParse(a.metadata?['matchRank']?.toString() ?? '') ?? 5; - final bRank = - int.tryParse(b.metadata?['matchRank']?.toString() ?? '') ?? 5; - if (aRank != bRank) return aRank.compareTo(bRank); - - final aName = a.canonicalName.toLowerCase(); - final bName = b.canonicalName.toLowerCase(); - final target = cleanPhrase.toLowerCase(); - if (aName == target && bName != target) return -1; - if (bName == target && aName != target) return 1; - if (aName.startsWith(target) && !bName.startsWith(target)) return -1; - if (bName.startsWith(target) && !aName.startsWith(target)) return 1; - return 0; - }); - - return candidates.take(limit).toList(); - } - - @override - Future> lookupStages( - String phrase, { - String? eventId, - String? eventName, - int limit = 25, - }) async { - final clean = phrase.trim().replaceAll("'", "''"); - if (clean.isEmpty) return []; - - final cleanLower = clean.toLowerCase(); - final cleanNum = cleanLower - .replaceAll('ss', '') - .replaceAll('stage', '') - .trim(); - - final patterns = _buildCandidatePatterns(phrase); - final patternClauses = []; - for (final p in patterns) { - final pEscaped = p.replaceAll("'", "''").toLowerCase(); - patternClauses.add("LOWER(stg.stage_name) LIKE '%$pEscaped%'"); - if (pEscaped.length >= 3 && - !pEscaped.contains(' ') && - !pEscaped.endsWith('%')) { - patternClauses.add( - "REPLACE(LOWER(stg.stage_name), ' ', '') LIKE '%$pEscaped%'", - ); - } - } - - patternClauses.add("stg.stage_number = '$clean'"); - if (cleanNum.isNotEmpty) { - patternClauses.add("stg.stage_number = '$cleanNum'"); - } - - final whereClauses = ['(${patternClauses.join(' OR ')})']; - - if (eventId != null && eventId.isNotEmpty) { - whereClauses.add("stg.event_id = '${eventId.replaceAll("'", "''")}'"); - } else if (eventName != null && eventName.isNotEmpty) { - final sanitizedEv = eventName.replaceAll("'", "''").toLowerCase(); - whereClauses.add("LOWER(ev.event_name) LIKE '%$sanitizedEv%'"); - } - - final whereSql = 'WHERE ${whereClauses.join(' AND ')}'; - final sql = - ''' - SELECT - stg.stage_id, - stg.stage_name, - stg.stage_number, - stg.event_id, - ev.event_name - FROM rally_stages stg - LEFT JOIN rally_events ev ON stg.event_id = ev.event_id - $whereSql - ORDER BY - CASE - WHEN LOWER(stg.stage_name) = '$cleanLower' THEN 1 - WHEN LOWER(stg.stage_name) LIKE '$cleanLower%' THEN 2 - ELSE 3 - END - LIMIT $limit; - '''; - - final rows = await _dbService.query(sql); - return rows.map((r) { - final stageId = r['stage_id']?.toString() ?? ''; - final stageName = r['stage_name']?.toString() ?? ''; - final stageNum = r['stage_number']?.toString(); - final evName = r['event_name']?.toString(); - final evId = r['event_id']?.toString(); - - final parts = []; - if (evName != null && evName.isNotEmpty) parts.add(evName); - if (stageNum != null && stageNum.isNotEmpty) parts.add('SS$stageNum'); - - return EntityCandidate( - id: stageId, - type: EntityType.stage, - canonicalName: stageName, - subtitle: parts.isNotEmpty ? parts.join(' • ') : null, - metadata: { - 'stageNumber': stageNum, - 'eventId': evId, - 'eventName': evName, - }, - ); - }).toList(); - } - - @override - Future> lookupCities( - String phrase, { - String? country, - int limit = 25, - }) async { - final clean = phrase.trim().replaceAll("'", "''"); - if (clean.isEmpty) return []; - - final cleanLower = clean.toLowerCase(); - final whereClauses = [ - "LOWER(ev.city) LIKE '%$cleanLower%'", - "ev.city IS NOT NULL", - "TRIM(ev.city) != ''", - ]; - - if (country != null && - country.trim().isNotEmpty && - country.toUpperCase() != 'ALL') { - final sanitizedCountry = country - .trim() - .replaceAll("'", "''") - .toLowerCase(); - whereClauses.add("LOWER(ev.country) LIKE '%$sanitizedCountry%'"); - } - - final whereSql = 'WHERE ${whereClauses.join(' AND ')}'; - final sql = - ''' - SELECT DISTINCT - ev.city, - ev.country - FROM rally_events ev - $whereSql - LIMIT $limit; - '''; - - final rows = await _dbService.query(sql); - return rows.map((r) { - final city = r['city']?.toString() ?? ''; - final cCountry = r['country']?.toString(); - - return EntityCandidate( - id: 'city_${city.toLowerCase().replaceAll(' ', '_')}', - type: EntityType.city, - canonicalName: city, - subtitle: cCountry != null && cCountry.isNotEmpty - ? cCountry.toUpperCase() - : null, - metadata: {'country': cCountry}, - ); - }).toList(); - } - - @override - Future> lookupUploaders( - String phrase, { - int limit = 25, - }) async { - final clean = phrase.trim().replaceAll("'", "''"); - if (clean.isEmpty) return []; - - final cleanLower = clean.toLowerCase(); - final sql = - ''' - SELECT - fp.fan_id AS id, - ua.user_name AS username, - fp.full_name, - ua.email, - fp.profile_picture - FROM user_fan_profile fp - LEFT JOIN user_account ua ON fp.account_id = ua.id - WHERE LOWER(fp.full_name) LIKE '%$cleanLower%' - OR LOWER(ua.user_name) LIKE '%$cleanLower%' - OR LOWER(ua.email) LIKE '%$cleanLower%' - ORDER BY - CASE - WHEN LOWER(fp.full_name) = '$cleanLower' OR LOWER(ua.user_name) = '$cleanLower' THEN 1 - WHEN LOWER(fp.full_name) LIKE '$cleanLower%' OR LOWER(ua.user_name) LIKE '$cleanLower%' THEN 2 - ELSE 3 - END - LIMIT $limit; - '''; - - final rows = await _dbService.query(sql); - return rows.map((r) { - final id = r['id']?.toString() ?? ''; - final fullName = r['full_name']?.toString()?.trim(); - final username = r['username']?.toString()?.trim(); - final email = r['email']?.toString()?.trim(); - final profilePic = r['profile_picture']?.toString(); - - final displayName = (username != null && username.isNotEmpty) - ? username - : ((fullName != null && fullName.isNotEmpty) - ? fullName - : ((email != null && email.isNotEmpty) - ? email - : 'Rally Contributor')); - - return EntityCandidate( - id: id, - type: EntityType.uploader, - canonicalName: displayName, - subtitle: - (fullName != null && fullName.isNotEmpty && fullName != username) - ? fullName - : (username != null && username.isNotEmpty ? '@$username' : null), - metadata: { - 'username': username, - 'fullName': fullName, - 'fanId': id, - 'profilePicture': profilePic, - }, - ); - }).toList(); - } -} diff --git a/lib/services/llm/natural_language_search_service.dart b/lib/services/llm/natural_language_search_service.dart index dc6c264..bbf8aad 100644 --- a/lib/services/llm/natural_language_search_service.dart +++ b/lib/services/llm/natural_language_search_service.dart @@ -202,12 +202,11 @@ class NaturalLanguageSearchService { NaturalLanguageSearchService({ required this.parser, required this.entityResolver, - ISearchRepository? repository, + required this.repository, VoiceEntityRecoveryService? voiceRecoveryService, SpecialQueryMatcher? specialQueryMatcher, FriendlyResponseService? friendlyResponses, - }) : repository = repository ?? SearchRepository(), - voiceRecoveryService = + }) : voiceRecoveryService = voiceRecoveryService ?? const VoiceEntityRecoveryService(), specialQueryMatcher = specialQueryMatcher ?? const SpecialQueryMatcher(), friendlyResponses = friendlyResponses ?? const FriendlyResponseService(); diff --git a/lib/services/offline/offline_bootstrap.dart b/lib/services/offline/offline_bootstrap.dart index 43a2c45..0f0c741 100644 --- a/lib/services/offline/offline_bootstrap.dart +++ b/lib/services/offline/offline_bootstrap.dart @@ -5,7 +5,7 @@ import 'package:sqflite/sqflite.dart'; import 'offline_database.dart'; import 'offline_search_engine.dart'; -import 'offline_search_router.dart'; +import '../latency/search_latency_coordinator.dart'; import 'offline_snapshot_sync.dart'; /// Reachability signal backed by `connectivity_plus`. A best-effort hint only — diff --git a/lib/services/offline/offline_messaging.dart b/lib/services/offline/offline_messaging.dart index e189976..8ea7a0a 100644 --- a/lib/services/offline/offline_messaging.dart +++ b/lib/services/offline/offline_messaging.dart @@ -6,6 +6,7 @@ enum OfflineUxState { offlineLocalResults, offlineStaleResults, lowBandwidthLocalFallback, + lateOnlineResultAvailable, backendUnreachableLocalAvailable, backendUnreachableLocalUnsupported, noLocalSnapshot, @@ -49,10 +50,15 @@ class OfflineMessagingService { 'Show results; offer refresh when online', ), OfflineUxState.lowBandwidthLocalFallback: OfflineMessage( - 'Bit of a slow stage out there…', - "We're using local rally data while the connection catches up.", + 'Taking the service road', + 'Showing saved rally data while the connection catches up.', 'Show local now; keep trying online', ), + OfflineUxState.lateOnlineResultAvailable: OfflineMessage( + 'HQ has fresh results', + 'The full search finished. Saved data is on screen until you switch.', + 'Show latest', + ), OfflineUxState.backendUnreachableLocalAvailable: OfflineMessage( "The pit crew can't reach HQ right now", "You're still searching with the data saved on this device.", diff --git a/lib/services/offline/offline_search_router.dart b/lib/services/offline/offline_search_router.dart deleted file mode 100644 index f58f161..0000000 --- a/lib/services/offline/offline_search_router.dart +++ /dev/null @@ -1,125 +0,0 @@ -import 'dart:async'; - -import 'offline_messaging.dart'; -import 'offline_search_engine.dart'; - -/// Reachability signal. Abstracted so the policy is testable without a device. -abstract class ConnectivityProbe { - Future isOnline(); -} - -/// How a routed search was answered. -enum RouteMode { - onlineAuthoritative, - offlineLocal, // device plainly offline - lowBandwidthLocal, // online attempted, budget elapsed -> local shown, online still running - backendUnreachableLocal, // online attempted and failed -> local -} - -class RouteResult { - final RouteMode mode; - final O? online; - final OfflineSearchOutcome? offline; - - /// When [mode] is [RouteMode.lowBandwidthLocal], the still-running online - /// request. The caller must NOT swap silently — offer an explicit - /// "HQ answered — show latest" affordance when it completes. - final Future? pendingOnline; - final OfflineUxState uxState; - - const RouteResult({ - required this.mode, - this.online, - this.offline, - this.pendingOnline, - required this.uxState, - }); -} - -/// NETWORK_FIRST_WITH_LOCAL_FALLBACK. -/// -/// Accuracy > latency: the authoritative online pipeline is tried first when the -/// device looks reachable, within a bounded, bandwidth-aware fallback budget. On -/// a plainly-offline device the online attempt is skipped entirely. A local -/// result never gets silently replaced by a late online result — promotion is -/// only ever offered via an explicit affordance. -/// -/// The budget numbers are deliberately conservative and tunable; the shape is -/// fixed, the numbers are not. -class OfflineSearchRouter { - final ConnectivityProbe connectivity; - final OfflineSearchEngine engine; - - /// Bandwidth-aware fallback budget — materially shorter than the full request - /// timeout (35 s). When it elapses without an online answer, the local result - /// is surfaced immediately while the online request keeps running. - final Duration fallbackBudget; - - const OfflineSearchRouter({ - required this.connectivity, - required this.engine, - this.fallbackBudget = const Duration(seconds: 4), - }); - - /// Routes one query. [online] performs the authoritative online search; - /// [rawText] is used for the local fallback. - Future> route({ - required String rawText, - required Future Function() online, - int limit = 20, - int offset = 0, - }) async { - Future local() => engine.search(rawText, limit: limit, offset: offset); - - // A. Plainly offline -> local immediately (no wasted wait). - final online0 = await connectivity.isOnline(); - if (!online0) { - final outcome = await local(); - return RouteResult( - mode: RouteMode.offlineLocal, - offline: outcome, - uxState: _stateForOffline(outcome), - ); - } - - // B/C. Try online within the bandwidth-aware budget. - final onlineFuture = online(); - try { - final result = await onlineFuture.timeout(fallbackBudget); - return RouteResult(mode: RouteMode.onlineAuthoritative, online: result, uxState: OfflineUxState.online); - } on TimeoutException { - // Budget elapsed: surface local now, keep online running for promotion. - final outcome = await local(); - // Prevent an unhandled error if the still-running request later fails. - final guarded = onlineFuture.catchError((Object e) => throw e); - return RouteResult( - mode: RouteMode.lowBandwidthLocal, - offline: outcome, - pendingOnline: guarded, - uxState: OfflineUxState.lowBandwidthLocalFallback, - ); - } catch (_) { - // Backend error / network drop -> deterministic local fallback. - final outcome = await local(); - return RouteResult( - mode: RouteMode.backendUnreachableLocal, - offline: outcome, - uxState: OfflineUxState.backendUnreachableLocalAvailable, - ); - } - } - - OfflineUxState _stateForOffline(OfflineSearchOutcome outcome) { - switch (outcome.kind) { - case OfflineOutcomeKind.clarification: - return OfflineUxState.offlineAmbiguity; - case OfflineOutcomeKind.noMatch: - return OfflineUxState.offlineSafeNoMatch; - case OfflineOutcomeKind.unsupported: - return OfflineUxState.offlineQueryUnsupported; - case OfflineOutcomeKind.special: - case OfflineOutcomeKind.results: - return OfflineUxState.offlineLocalResults; - } - } -} diff --git a/lib/services/python_search_api_client.dart b/lib/services/python_search_api_client.dart index e0d02d3..bd38638 100644 --- a/lib/services/python_search_api_client.dart +++ b/lib/services/python_search_api_client.dart @@ -18,6 +18,7 @@ import '../models/supported_language.dart'; import '../models/video_action.dart'; import 'search_repository.dart'; import 'friendly_response_service.dart'; +import 'latency/latency_policy.dart'; import 'llm/natural_language_search_service.dart'; import 'llm/query_parse_result.dart'; @@ -31,12 +32,17 @@ class SearchBackendConfig { final Duration voiceTimeout; final Map headers; - const SearchBackendConfig({ + /// Timeouts come from [LatencyPolicy] rather than being declared here, so + /// the client and the fallback coordinator can never disagree about how long + /// an online request is allowed to take. + SearchBackendConfig({ this.pythonBaseUrl, - this.typedTimeout = const Duration(seconds: 35), - this.voiceTimeout = const Duration(seconds: 75), + LatencyPolicy policy = LatencyPolicy.standard, + Duration? typedTimeout, + Duration? voiceTimeout, this.headers = const {}, - }); + }) : typedTimeout = typedTimeout ?? policy.overallOnlineTimeout, + voiceTimeout = voiceTimeout ?? policy.overallVoiceTimeout; factory SearchBackendConfig.fromEnvironment() { final url = dotenv.isInitialized @@ -102,13 +108,31 @@ class PythonApiException implements Exception { class PythonConversationResponse { final SearchConversationSession session; final NaturalLanguageSearchResult result; + + /// The client's session generation counter, echoed by the backend. Used to + /// reject a response that belongs to a superseded search. final int? requestId; + + /// The end-to-end correlation id (`X-Request-Id`). Joins this response to + /// the backend's structured timing line for the same request. + final String? traceId; + + /// Measured wall time of the HTTP call, for client-side latency records. + final int? networkRoundtripMs; + + /// Backend phase breakdown, present only when the backend runs with debug + /// timings enabled. Never shown to users. + final Map? backendTimings; + final SpeechTranscriptionResult? transcription; final Map telemetry; const PythonConversationResponse({ required this.session, required this.result, this.requestId, + this.traceId, + this.networkRoundtripMs, + this.backendTimings, this.transcription, this.telemetry = const {}, }); @@ -145,6 +169,9 @@ class CloudTranscriptionResponse { } class PythonSearchApiClient { + /// End-to-end correlation header, matched by the backend timing middleware. + static const String requestIdHeader = 'X-Request-Id'; + final Uri baseUrl; final http.Client _http; final Duration typedTimeout; @@ -154,10 +181,13 @@ class PythonSearchApiClient { PythonSearchApiClient({ required this.baseUrl, http.Client? httpClient, - this.typedTimeout = const Duration(seconds: 35), - this.voiceTimeout = const Duration(seconds: 75), + LatencyPolicy policy = LatencyPolicy.standard, + Duration? typedTimeout, + Duration? voiceTimeout, this.headers = const {}, - }) : _http = httpClient ?? http.Client(); + }) : _http = httpClient ?? http.Client(), + typedTimeout = typedTimeout ?? policy.overallOnlineTimeout, + voiceTimeout = voiceTimeout ?? policy.overallVoiceTimeout; factory PythonSearchApiClient.fromConfig( SearchBackendConfig config, { @@ -210,14 +240,25 @@ class PythonSearchApiClient { required SearchConversationSession session, required String language, required int requestId, + String? traceId, }) async { - final body = await _postJson('/v1/conversation/search', { - 'query': query, - 'session': _sessionToJson(session), - 'language': language, - 'requestId': requestId, - }, typedTimeout); - return _decodeConversation(body); + final stopwatch = Stopwatch()..start(); + final body = await _postJson( + '/v1/conversation/search', + { + 'query': query, + 'session': _sessionToJson(session), + 'language': language, + 'requestId': requestId, + }, + typedTimeout, + traceId: traceId, + ); + stopwatch.stop(); + return _decodeConversation( + body, + networkRoundtripMs: stopwatch.elapsedMilliseconds, + ); } Future voice({ @@ -227,7 +268,9 @@ class PythonSearchApiClient { required String language, required int requestId, String? editedTranscript, + String? traceId, }) async { + final stopwatch = Stopwatch()..start(); final uri = _uri('/v1/voice/search', { 'filename': filename, 'language': language, @@ -240,13 +283,21 @@ class PythonSearchApiClient { () => _http .post( uri, - headers: {...headers, 'Content-Type': 'application/octet-stream'}, + headers: { + ...headers, + 'Content-Type': 'application/octet-stream', + requestIdHeader: ?traceId, + }, body: audioBytes, ) .timeout(voiceTimeout), ); + stopwatch.stop(); final body = _decodeBody(response); - final decoded = _decodeConversation(body); + final decoded = _decodeConversation( + body, + networkRoundtripMs: stopwatch.elapsedMilliseconds, + ); final raw = Map.from( body['transcription'] as Map? ?? const {}, ); @@ -285,6 +336,9 @@ class PythonSearchApiClient { session: decoded.session, result: decoded.result, requestId: decoded.requestId, + traceId: decoded.traceId, + networkRoundtripMs: decoded.networkRoundtripMs, + backendTimings: decoded.backendTimings, transcription: transcription, telemetry: Map.from( body['telemetry'] as Map? ?? const {}, @@ -295,13 +349,20 @@ class PythonSearchApiClient { Future> _postJson( String path, Map payload, - Duration timeout, - ) async { + Duration timeout, { + String? traceId, + }) async { final response = await _send( () => _http .post( _uri(path), - headers: {...headers, 'Content-Type': 'application/json'}, + headers: { + ...headers, + 'Content-Type': 'application/json', + // Correlation only: an opaque id the backend echoes into its + // structured timing line. Carries no user or query content. + requestIdHeader: ?traceId, + }, body: jsonEncode(payload), ) .timeout(timeout), @@ -350,7 +411,10 @@ class PythonSearchApiClient { Uri _uri(String path, [Map? query]) => baseUrl.resolve(path).replace(queryParameters: query); - PythonConversationResponse _decodeConversation(Map body) { + PythonConversationResponse _decodeConversation( + Map body, { + int? networkRoundtripMs, + }) { final resultMap = Map.from( body['result'] as Map? ?? const {}, ); @@ -415,6 +479,11 @@ class PythonSearchApiClient { session: session, result: result, requestId: _nullableInt(body['requestId']), + traceId: body['traceId']?.toString(), + networkRoundtripMs: networkRoundtripMs, + backendTimings: resultMap['timings'] is Map + ? Map.from(resultMap['timings'] as Map) + : null, ); } diff --git a/lib/services/search_repository.dart b/lib/services/search_repository.dart index eecb8e7..13dd5da 100644 --- a/lib/services/search_repository.dart +++ b/lib/services/search_repository.dart @@ -1,10 +1,10 @@ -import '../models/search_intent.dart'; import '../models/search_query.dart'; import '../models/search_results.dart'; import '../models/video_action.dart'; -import '../models/video_action_search_query.dart'; -import 'database_service.dart'; +/// Contract implemented by the FastAPI-backed [PythonSearchRepository] +/// (see python_search_api_client.dart). The device never talks to the +/// database directly; all search goes over HTTPS to the backend. abstract class ISearchRepository { /// General dispatch method executing any search query and returning typed response Future> search(SearchQuery query); @@ -19,178 +19,3 @@ abstract class ISearchRepository { Future> getTopUploaders(SearchQuery query); Future> getTopDriversByWins(SearchQuery query); } - -class SearchRepository implements ISearchRepository { - final DatabaseService _dbService; - - SearchRepository({DatabaseService? dbService}) - : _dbService = dbService ?? DatabaseService(); - - @override - Future> search(SearchQuery query) async { - switch (query.intent) { - case SearchIntent.searchRallies: - return await searchRallies(query); - case SearchIntent.searchDriverRallies: - return await searchDriverRallies(query); - case SearchIntent.searchDriverWins: - return await searchDriverWins(query); - case SearchIntent.getRallyResults: - return await getRallyResults(query); - case SearchIntent.getRallyTopFinishers: - return await getRallyTopFinishers(query); - case SearchIntent.searchVideoActions: - return await searchVideoActions(query); - case SearchIntent.searchDriverVideos: - return await searchDriverVideos(query); - case SearchIntent.getTopUploaders: - return await getTopUploaders(query); - case SearchIntent.getTopDriversByWins: - return await getTopDriversByWins(query); - } - } - - @override - Future> searchRallies(SearchQuery query) async { - final count = await _dbService.countRallies(query); - final rows = await _dbService.searchRallies(query); - final results = rows.map((r) => RallySearchResult.fromMap(r)).toList(); - - return SearchResponse( - intent: SearchIntent.searchRallies, - results: results, - totalCount: count, - hasMore: (query.offset + results.length) < count, - limit: query.limit, - offset: query.offset, - ); - } - - @override - Future> searchDriverRallies(SearchQuery query) async { - final count = await _dbService.countDriverRallies(query); - final rows = await _dbService.searchDriverRallies(query); - final results = rows.map((r) => RallyParticipationResult.fromMap(r)).toList(); - - return SearchResponse( - intent: SearchIntent.searchDriverRallies, - results: results, - totalCount: count, - hasMore: (query.offset + results.length) < count, - limit: query.limit, - offset: query.offset, - ); - } - - @override - Future> searchDriverWins(SearchQuery query) async { - final count = await _dbService.countDriverWins(query); - final rows = await _dbService.searchDriverWins(query); - final results = rows.map((r) => RallyParticipationResult.fromMap(r)).toList(); - - return SearchResponse( - intent: SearchIntent.searchDriverWins, - results: results, - totalCount: count, - hasMore: (query.offset + results.length) < count, - limit: query.limit, - offset: query.offset, - ); - } - - @override - Future> getRallyResults(SearchQuery query) async { - final rows = await _dbService.getRallyResults(query); - final results = rows.map((r) => RallyResult.fromMap(r)).toList(); - - return SearchResponse( - intent: SearchIntent.getRallyResults, - results: results, - totalCount: results.length, - hasMore: false, - limit: query.limit, - offset: query.offset, - ); - } - - @override - Future> getRallyTopFinishers(SearchQuery query) async { - final count = await _dbService.countRallyTopFinishers(query); - final rows = await _dbService.getRallyTopFinishers(query); - final results = rows.map((r) => RallyResult.fromMap(r)).toList(); - - return SearchResponse( - intent: SearchIntent.getRallyTopFinishers, - results: results, - totalCount: count, - hasMore: (query.offset + results.length) < count, - limit: query.limit, - offset: query.offset, - ); - } - - @override - Future> searchVideoActions(SearchQuery query) async { - final count = await _dbService.countVideoActions(query); - final rows = await _dbService.searchVideoActions(query); - final results = rows.map((r) => VideoAction.fromMap(r)).toList(); - - return SearchResponse( - intent: SearchIntent.searchVideoActions, - results: results, - totalCount: count, - hasMore: (query.offset + results.length) < count, - limit: query.limit, - offset: query.offset, - ); - } - - - @override - Future> searchDriverVideos(SearchQuery query) async { - final count = await _dbService.countDriverVideos(query); - final rows = await _dbService.searchDriverVideos(query); - final results = rows.map((r) => VideoSearchResult.fromMap(r)).toList(); - - return SearchResponse( - intent: SearchIntent.searchDriverVideos, - results: results, - totalCount: count, - hasMore: (query.offset + results.length) < count, - limit: query.limit, - offset: query.offset, - ); - } - - @override - Future> getTopUploaders(SearchQuery query) async { - final count = await _dbService.countTopUploaders(query); - final rows = await _dbService.getTopUploaders(query); - final results = rows.map((r) => UploaderSearchResult.fromMap(r)).toList(); - - return SearchResponse( - intent: SearchIntent.getTopUploaders, - results: results, - totalCount: count, - hasMore: (query.offset + results.length) < count, - limit: query.limit, - offset: query.offset, - ); - } - - @override - Future> getTopDriversByWins(SearchQuery query) async { - final count = await _dbService.countTopDriversByWins(query); - final rows = await _dbService.getTopDriversByWins(query); - final results = rows.map((r) => DriverWinResult.fromMap(r)).toList(); - - return SearchResponse( - intent: SearchIntent.getTopDriversByWins, - results: results, - totalCount: count, - hasMore: (query.offset + results.length) < count, - limit: query.limit, - offset: query.offset, - ); - } -} diff --git a/lib/services/video_action_repository.dart b/lib/services/video_action_repository.dart deleted file mode 100644 index 3d8f96a..0000000 --- a/lib/services/video_action_repository.dart +++ /dev/null @@ -1,110 +0,0 @@ -import '../models/video_action.dart'; -import '../models/video_action_search_query.dart'; -import 'database_service.dart'; - -abstract class IVideoActionRepository { - Future> getVideoActionsForVideo( - int videoId, { - String? defaultVideoUrl, - int? defaultStreamId, - double? defaultClipStartTime, - double? defaultClipDuration, - }); - - Future> getVideoActionsForStream( - int streamId, { - String? defaultVideoUrl, - double? defaultClipStartTime, - double? defaultClipDuration, - }); - - Future> getRecentVideoActions({ - int limit = 20, - int offset = 0, - String? actionType, - }); - - Future> searchVideoActions( - VideoActionSearchQuery query, - ); - - Future countVideoActions( - VideoActionSearchQuery query, - ); -} - -class VideoActionRepository implements IVideoActionRepository { - final DatabaseService _dbService; - - VideoActionRepository({DatabaseService? dbService}) - : _dbService = dbService ?? DatabaseService(); - - @override - Future> getVideoActionsForVideo( - int videoId, { - String? defaultVideoUrl, - int? defaultStreamId, - double? defaultClipStartTime, - double? defaultClipDuration, - }) async { - final rows = await _dbService.getVideoActionsForVideo(videoId); - return rows - .map((row) => VideoAction.fromMap( - row, - defaultVideoUrl: defaultVideoUrl, - defaultStreamId: defaultStreamId, - defaultClipStartTime: defaultClipStartTime, - defaultClipDuration: defaultClipDuration, - )) - .toList(); - } - - @override - Future> getVideoActionsForStream( - int streamId, { - String? defaultVideoUrl, - double? defaultClipStartTime, - double? defaultClipDuration, - }) async { - final rows = await _dbService.getVideoActionsForStream(streamId); - return rows - .map((row) => VideoAction.fromMap( - row, - defaultVideoUrl: defaultVideoUrl, - defaultStreamId: streamId, - defaultClipStartTime: defaultClipStartTime, - defaultClipDuration: defaultClipDuration, - )) - .toList(); - } - - @override - Future> getRecentVideoActions({ - int limit = 20, - int offset = 0, - String? actionType, - }) async { - final rows = await _dbService.getRecentVideoActions( - limit: limit, - offset: offset, - actionType: actionType, - ); - return rows.map((row) => VideoAction.fromMap(row)).toList(); - } - - @override - Future> searchVideoActions( - VideoActionSearchQuery query, - ) async { - final rows = await _dbService.searchVideoActions(query); - return rows.map((row) => VideoAction.fromMap(row)).toList(); - } - - @override - Future countVideoActions( - VideoActionSearchQuery query, - ) async { - return await _dbService.countVideoActions(query); - } -} - diff --git a/pubspec.lock b/pubspec.lock index c3a02ea..b49d578 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -25,14 +25,6 @@ packages: url: "https://pub.dev" source: hosted version: "2.1.2" - buffer: - dependency: transitive - description: - name: buffer - sha256: "389da2ec2c16283c8787e0adaede82b1842102f8c8aae2f49003a766c5c6b3d1" - url: "https://pub.dev" - source: hosted - version: "1.2.3" characters: dependency: transitive description: @@ -42,7 +34,7 @@ packages: source: hosted version: "1.4.1" clock: - dependency: transitive + dependency: "direct main" description: name: clock sha256: fddb70d9b5277016c77a80201021d40a2247104d9f4aa7bab7157b7e3f05b84b @@ -317,14 +309,6 @@ packages: url: "https://pub.dev" source: hosted version: "1.19.0" - mysql_client: - dependency: "direct main" - description: - name: mysql_client - sha256: "6a0fdcbe3e0721c637f97ad24649be2f70dbce2b21ede8f962910e640f753fc2" - url: "https://pub.dev" - source: hosted - version: "0.0.27" native_toolchain_c: dependency: transitive description: @@ -666,14 +650,6 @@ packages: url: "https://pub.dev" source: hosted version: "0.7.12" - tuple: - dependency: transitive - description: - name: tuple - sha256: a97ce2013f240b2f3807bcbaf218765b6f301c3eff91092bcfa23a039e7dd151 - url: "https://pub.dev" - source: hosted - version: "2.0.2" typed_data: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index 46320a0..f7b14f7 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -39,7 +39,10 @@ dependencies: cupertino_icons: ^1.0.8 flutter_dotenv: ^6.0.1 http: ^1.2.0 - mysql_client: ^0.0.27 + # Zone-overridable clock. Search latency is measured through it so the same + # code reports real durations in production and fake-clock durations under + # widget tests, instead of a Stopwatch that only ever sees wall time. + clock: ^1.1.1 video_player: ^2.14.0 record: ^6.2.0 speech_to_text: ^7.4.0 @@ -77,9 +80,10 @@ flutter: # the material Icons class. uses-material-design: true - # To add assets to your application, add an assets section, like this: + # Public, non-sensitive client configuration only. The device never bundles + # server secrets (DB credentials, API keys) — those stay server-side. assets: - - .env + - assets/config/app_config.env # An image asset can refer to one or more resolution-specific "variants", see # https://flutter.dev/to/resolution-aware-images diff --git a/test/db_analyzer.dart b/test/db_analyzer.dart deleted file mode 100644 index b8ff2f7..0000000 --- a/test/db_analyzer.dart +++ /dev/null @@ -1,59 +0,0 @@ -// ignore_for_file: avoid_print -import 'package:flutter_test/flutter_test.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:ai_rally_search/services/database_service.dart'; - -void main() { - TestWidgetsFlutterBinding.ensureInitialized(); - - test('Inspect rally_events countries', () async { - await dotenv.load(fileName: '.env'); - final db = DatabaseService(); - final conn = await db.connect(); - - // 1. Describe table - final desc = await conn.execute('DESCRIBE rally_events;'); - print('\n=== rally_events SCHEMA ==='); - for (final row in desc.rows) { - final map = row.assoc(); - print('${map['Field']} (${map['Type']})'); - } - - // 2. Distinct countries with event counts - final countryCounts = await conn.execute(''' - SELECT country, COUNT(*) as event_count - FROM rally_events - GROUP BY country - ORDER BY country; - '''); - print('\n=== DISTINCT COUNTRIES & EVENT COUNTS IN rally_events ==='); - for (final row in countryCounts.rows) { - final map = row.assoc(); - print('Country: "${map['country']}" | Events: ${map['event_count']}'); - } - - // 3. Inspect specific rows that have short/unusual country codes or names - final sampleRows = await conn.execute(''' - SELECT event_id, event_name, country, city, YEAR(start_date) as year - FROM rally_events - WHERE country IN ('at', 'es', 'gb', 'hr', 'ie', 'ke', 'lv', 'none', 'qa', 'Scotland', 'Wales') - OR country IS NULL OR TRIM(country) = ''; - '''); - print('\n=== SPECIAL / 2-LETTER / OUTLIER ROWS IN rally_events ==='); - for (final row in sampleRows.rows) { - final map = row.assoc(); - print('ID: ${map['event_id']} | Name: "${map['event_name']}" | Year: ${map['year']} | City: "${map['city']}" | Country: "${map['country']}"'); - } - - // 4. Sample check on total rally events - final total = await conn.execute('SELECT COUNT(*) as total FROM rally_events;'); - print('\nTotal rally_events: ${total.rows.first.assoc()['total']}'); - - await db.close(); - }); -} - - - - - diff --git a/test/db_connection_test.dart b/test/db_connection_test.dart deleted file mode 100644 index cd77ea8..0000000 --- a/test/db_connection_test.dart +++ /dev/null @@ -1,23 +0,0 @@ -// ignore_for_file: avoid_print -import 'package:flutter_test/flutter_test.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:ai_rally_search/services/database_service.dart'; - -void main() { - TestWidgetsFlutterBinding.ensureInitialized(); - - test('AWS RDS MySQL Database Connection Test', () async { - await dotenv.load(fileName: '.env'); - final dbService = DatabaseService(); - - final status = await dbService.testConnection(); - print('DB Connection Status: $status'); - - expect(status['success'], isTrue, - reason: 'Failed to connect: ${status['error']}'); - expect(status['tableCount'], greaterThan(0)); - print('Successfully verified ${status['tableCount']} tables in AWS RDS!'); - - await dbService.close(); - }); -} diff --git a/test/eval/audit_misses_test.dart b/test/eval/audit_misses_test.dart deleted file mode 100644 index 493dc58..0000000 --- a/test/eval/audit_misses_test.dart +++ /dev/null @@ -1,186 +0,0 @@ -import 'package:flutter_test/flutter_test.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:ai_rally_search/services/database_service.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/entity_lookup_repository.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/database_entity_resolver.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/phonetic_matching_helper.dart'; -import 'package:ai_rally_search/models/search_query.dart'; -import 'package:ai_rally_search/models/search_intent.dart'; - -void main() { - TestWidgetsFlutterBinding.ensureInitialized(); - - test('Audit exact misses and non-top-1 cases', () async { - await dotenv.load(fileName: '.env'); - final dbService = DatabaseService(); - final lookupRepo = DatabaseEntityLookupRepository(dbService: dbService); - final resolver = DatabaseEntityResolver( - repository: lookupRepo, - minConfidenceThreshold: 0.75, - minScoreGap: 0.15, - ); - - final rallyCases = [ - {'canonical': 'Rally Alūksne 2026', 'perturbations': ['Rally Aluske', 'aluxne', 'aluksne', 'Rally Aluksne']}, - {'canonical': '6 Uren van Kortrijk 2024', 'perturbations': ['kortrik', '6 Uren van Kortrik', 'Kortrijk 2024', 'Uren van Kortrijk']}, - {'canonical': 'Rali Serras de Fafe 2025', 'perturbations': ['Rali Serras de Faf', 'Serras de Fafe', 'Rally Fafe', 'Fafe 2025']}, - {'canonical': '7bet Rally Lazdijai 2025', 'perturbations': ['lazdiai', '7bet Rally Lazdiai', 'Rally Lazdijai', 'Lazdijai 2025']}, - {'canonical': "Rali Terras d'Aboboreira 2026", 'perturbations': ["aboborera", "Terras d'Aboboreira", "Rali Terras d Aboboreira 2026", "Aboboreira 2026"]}, - {'canonical': 'Polski Rajd Legend 2026', 'perturbations': ['Polski Raid Legend', 'Rajd Legend', 'Polski Rajd Legend', 'Polanica Legend']}, - {'canonical': 'Rally Vranov 2026', 'perturbations': ['Rally Vranow', 'Vranov 2026', 'Rally Vranov', 'Vranov Nad Toplou']}, - {'canonical': 'OBM Land der 1000 Hügel Rallye 2026', 'perturbations': ['1000 Hugel Rallye', 'Land der 1000 Hugel', '1000 Hügel', 'Hugel Rallye 2026']}, - {'canonical': 'Rallijsprints Cesavine 2026', 'perturbations': ['Cesavine', 'Rallijsprint Cesavine', 'Cesavine 2026', 'Cesavine Rally']}, - {'canonical': 'Rallye Régional des Ardennes 2025', 'perturbations': ['Regional des Ardennes', 'Rally Ardennes', 'Ardennes 2025', 'Rallye des Ardennes']}, - {'canonical': 'Century 21 Portugal Rally Series - Castelo Branco 2025', 'perturbations': ['Castelo Branco', 'Castelo Branco 2025', 'Rally Castelo Branco', 'Portugal Rally Series Castelo Branco']}, - {'canonical': 'Corrib Oil Galway International Rally 2026', 'perturbations': ['Galway Rally', 'Galway International 2026', 'Corrib Oil Galway', 'Galway 2026']}, - {'canonical': 'Assess Ireland International Rally of the Lakes 2026', 'perturbations': ['Rally of the Lakes', 'Rally of the Lakes 2026', 'International Rally of the Lakes', 'Lakes Rally 2026']}, - {'canonical': 'Clonakilty Park Hotel West Cork Rally 2026', 'perturbations': ['West Cork Rally', 'Westcork 2026', 'West Cork 2026', 'Clonakilty West Cork']}, - {'canonical': 'Samsonas Rally Fivemiletown 2026', 'perturbations': ['Fivemiletown', 'Fivemiletown Rally', 'Samsonas Fivemiletown', 'Fivemiletown 2026']}, - {'canonical': 'Modern Tyres Ulster Rally 2025', 'perturbations': ['Ulster Rally', 'Ulster Rally 2025', 'Modern Tyres Ulster', 'Ulster 2025']}, - {'canonical': 'Raven\'s Rock Stages Rally 2025', 'perturbations': ['Ravens Rock', 'Ravens Rock Stages', 'Ravens Rock 2025', 'Raven Rock Rally']}, - {'canonical': 'Birr Stages Rally 2026', 'perturbations': ['Birr Stages', 'Birr Rally', 'Birr Stages 2026', 'Birr 2026']}, - {'canonical': 'Fastnet Stages Rally 2025', 'perturbations': ['Fastnet Stages', 'Fastnet Rally', 'Fastnet 2025', 'Fastnet Stages 2025']}, - {'canonical': 'HK Cavan Stages Rally 2025', 'perturbations': ['Cavan Stages', 'Cavan Stages 2025', 'Cavan Rally', 'HK Cavan 2025']}, - ]; - - final driverCases = [ - {'canonical': 'Jon-Gunnar Støten', 'role': 'driver', 'perturbations': ['Jon Gunnar Stoten', 'Jon Gunnar Støten', 'Jon-Gunnar Stoten', 'Stoten']}, - {'canonical': 'Michal Babička', 'role': 'driver', 'perturbations': ['Michal Babicka', 'Michal Babicka', 'Babicka', 'Michal Babicka']}, - {'canonical': 'Adam Zelík', 'role': 'driver', 'perturbations': ['Adam Zelik', 'Adam Zelik', 'Zelik', 'Adam Zelik']}, - {'canonical': 'Věroslav Cvrček', 'role': 'driver', 'perturbations': ['Veroslav Cvrcek', 'Věroslav Cvrcek', 'Veroslav Cvrček', 'Cvrcek']}, - {'canonical': 'Piotr Krotoszyński', 'role': 'driver', 'perturbations': ['Piotr Krotoszynski', 'Piotr Krotoszynski', 'Krotoszynski', 'Piotr Krotoszynski']}, - {'canonical': 'Hervé Emeriau', 'role': 'driver', 'perturbations': ['Herve Emeriau', 'Hervé Emeriau', 'Herve Emerio', 'Emeriau']}, - {'canonical': 'José Paula', 'role': 'driver', 'perturbations': ['Jose Paula', 'José Paula', 'Jose Pawla', 'Paula']}, - {'canonical': 'Sergio Ramón Arrom', 'role': 'driver', 'perturbations': ['Sergio Ramon Arrom', 'Sergio Ramon', 'Ramon Arrom', 'Sergio Arrom']}, - {'canonical': 'Raphaël Czwartkowski', 'role': 'driver', 'perturbations': ['Raphael Czwartkowski', 'Raphael Czwartkovski', 'Czwartkowski', 'Raphaël Czwartkovski']}, - {'canonical': 'Vítor Matias', 'role': 'driver', 'perturbations': ['Vitor Matias', 'Vítor Matias', 'Vitor Mathias', 'Matias']}, - {'canonical': 'Stephen O\'Connor', 'role': 'driver', 'perturbations': ['Stephen OConnor', 'Stephen O\'Connor', 'Steven OConnor', 'Stephen O Connor']}, - {'canonical': 'Diarmuid O\'Toole', 'role': 'driver', 'perturbations': ['Diarmuid OToole', 'Diarmuid O\'Toole', 'Dermot OToole', 'Diarmuid O Toole']}, - {'canonical': 'Tanja Zingelmann-Hartjen', 'role': 'driver', 'perturbations': ['Tanja Zingelmann', 'Tanja Zingelmann Hartjen', 'Tanja Hartjen', 'Zingelmann-Hartjen']}, - {'canonical': 'Paweł Molgo', 'role': 'driver', 'perturbations': ['Pawel Molgo', 'Paweł Molgo', 'Pawel Malgo', 'Molgo']}, - {'canonical': 'Nenad Lončarič', 'role': 'driver', 'perturbations': ['Nenad Loncaric', 'Nenad Lončaric', 'Nenad Loncarich', 'Loncaric']}, - {'canonical': 'Matej Bogović', 'role': 'driver', 'perturbations': ['Matej Bogovic', 'Matej Bogović', 'Matej Bogovich', 'Bogovic']}, - {'canonical': 'Andrej Medić', 'role': 'driver', 'perturbations': ['Andrej Medic', 'Andrej Medić', 'Andrej Medich', 'Medic']}, - {'canonical': 'John Shanahan jnr.', 'role': 'driver', 'perturbations': ['John Shanahan', 'John Shanahan Jr', 'John Shanahan jnr', 'Shanahan']}, - {'canonical': 'Shea Breen', 'role': 'driver', 'perturbations': ['Shea Brean', 'Shea Breen', 'Shea Brain', 'Shay Breen']}, - {'canonical': 'Max Freeman', 'role': 'co_driver', 'perturbations': ['Max Freeman', 'Max Freman', 'Max Frieman', 'Freeman']}, - {'canonical': 'Jan-Erik Mäll', 'role': 'co_driver', 'perturbations': ['Jan Erik Mall', 'Jan-Erik Mall', 'Jan Erik Mäll', 'Mall']}, - {'canonical': 'Catharina Schmidt', 'role': 'co_driver', 'perturbations': ['Catharina Schmidt', 'Catherina Schmidt', 'Katarina Schmidt', 'Schmidt']}, - ]; - - final stageCases = [ - {'canonical': 'Woodstoxx Kemmelberg 1', 'perturbations': ['Kemelberg', 'Woodstoxx Kemelberg', 'Kemmelberg 1', 'Kemmelberg']}, - {'canonical': 'Duszniki - Zieleniec 2', 'perturbations': ['Dushniki', 'Duszniki Zieleniec', 'Duszniki', 'Zieleniec 2']}, - {'canonical': 'Seixoso 2', 'perturbations': ['Seixoso', 'Seixoso 2', 'Seiksozo', 'SS Seixoso']}, - {'canonical': 'Drumhallagh 2', 'perturbations': ['Drumhallagh', 'Drumhallagh 2', 'Drumhalagh', 'SS Drumhallagh']}, - {'canonical': 'Dikkebus 1', 'perturbations': ['Dikkebus', 'Dikebus', 'Dikkebus 1', 'SS Dikkebus']}, - {'canonical': 'Fafe 2Powerstage', 'perturbations': ['Fafe Powerstage', 'Fafe 2', 'Fafe', 'Powerstage Fafe']}, - {'canonical': 'Knockalla 2', 'perturbations': ['Knockalla', 'Knokalla', 'Knockalla 2', 'SS Knockalla']}, - {'canonical': 'Dunworley 2', 'perturbations': ['Dunworley', 'Dunworley 2', 'Dunworly', 'SS Dunworley']}, - {'canonical': 'Kellymount 1', 'perturbations': ['Kellymount', 'Kellymount 1', 'Kelley Mount', 'SS Kellymount']}, - {'canonical': 'Scart Mountain 1', 'perturbations': ['Scart Mountain', 'Scart Mountain 1', 'Scart Mt', 'SS Scart Mountain']}, - ]; - - print('AUDITING RECALL MISSES AND NON-TOP-1 CASES...'); - - // Audit driver misses specifically - for (final item in driverCases) { - final canonical = item['canonical'] as String; - final role = item['role'] as String; - final perturbations = item['perturbations'] as List; - - for (final p in perturbations) { - final candidates = await lookupRepo.lookupDrivers(p, limit: 50); - final scoredCandidates = candidates.map((c) { - final score = PhoneticMatchingHelper.computeCompositeScore( - queryPhrase: p, - candidateName: c.canonicalName, - isPerson: true, - ); - return c.copyWith(score: score); - }).toList() - ..sort((a, b) => (b.score ?? 0.0).compareTo(a.score ?? 0.0)); - - final inTop5 = scoredCandidates.take(5).any((c) => _matchesCanonical(c.canonicalName, canonical)); - final res = await resolver.resolve(SearchQuery(intent: SearchIntent.searchDriverVideos, driverName: p)); - final resolvedName = res.resolutions['driver']?.resolvedCandidate?.canonicalName; - final isResolvedCorrect = resolvedName != null && _matchesCanonical(resolvedName, canonical); - final clarContains = res.candidates.any((c) => _matchesCanonical(c.canonicalName, canonical)) || - (res.resolutions['driver']?.candidateOptions.any((c) => _matchesCanonical(c.canonicalName, canonical)) ?? false); - - if (!inTop5) { - print('\n[RECALL@5 MISS - $role]'); - print(' Canonical: "$canonical"'); - print(' Query/Perturbation: "$p"'); - print(' Top 5 Scored: ${scoredCandidates.take(5).map((c) => "${c.canonicalName} (${c.score?.toStringAsFixed(2)})").toList()}'); - print(' Total candidates returned: ${candidates.length}'); - print(' In full candidates list: ${candidates.any((c) => _matchesCanonical(c.canonicalName, canonical))}'); - } - - if (!isResolvedCorrect && !clarContains) { - print('\n[NON-TOP-1 MISS - $role]'); - print(' Canonical: "$canonical"'); - print(' Query: "$p"'); - print(' Resolution: resolved="$resolvedName", amb=${res.resolutions['driver']?.isAmbiguous}, strat=${res.resolutions['driver']?.strategy}, conf=${res.resolutions['driver']?.confidence}'); - } - } - } - - // Audit stage misses - for (final item in stageCases) { - final canonical = item['canonical'] as String; - final perturbations = item['perturbations'] as List; - - for (final p in perturbations) { - final candidates = await lookupRepo.lookupStages(p, limit: 35); - final scoredCandidates = candidates.map((c) { - final score = PhoneticMatchingHelper.computeCompositeScore( - queryPhrase: p, - candidateName: c.canonicalName, - ); - return c.copyWith(score: score); - }).toList() - ..sort((a, b) => (b.score ?? 0.0).compareTo(a.score ?? 0.0)); - - final inTop5 = scoredCandidates.take(5).any((c) => _matchesCanonical(c.canonicalName, canonical)); - final res = await resolver.resolve(SearchQuery(intent: SearchIntent.searchVideoActions, stageName: p)); - final resolvedName = res.resolutions['stage']?.resolvedCandidate?.canonicalName; - final isResolvedCorrect = resolvedName != null && _matchesCanonical(resolvedName, canonical); - final clarContains = res.candidates.any((c) => _matchesCanonical(c.canonicalName, canonical)) || - (res.resolutions['stage']?.candidateOptions.any((c) => _matchesCanonical(c.canonicalName, canonical)) ?? false); - - if (!inTop5) { - print('\n[STAGE RECALL@5 MISS]'); - print(' Canonical: "$canonical"'); - print(' Query: "$p"'); - print(' Top 5 Scored: ${scoredCandidates.take(5).map((c) => "${c.canonicalName} (${c.score?.toStringAsFixed(2)})").toList()}'); - } - - if (!isResolvedCorrect && !clarContains) { - print('\n[STAGE NON-TOP-1 MISS]'); - print(' Canonical: "$canonical"'); - print(' Query: "$p"'); - print(' Resolution: resolved="$resolvedName", amb=${res.resolutions['stage']?.isAmbiguous}, strat=${res.resolutions['stage']?.strategy}, conf=${res.resolutions['stage']?.confidence}'); - } - } - } - - await dbService.close(); - }, timeout: const Timeout(Duration(minutes: 2))); -} - -bool _matchesCanonical(String candidate, String target) { - final cNorm = PhoneticMatchingHelper.normalize(candidate); - final tNorm = PhoneticMatchingHelper.normalize(target); - if (cNorm == tNorm) return true; - - final cBase = PhoneticMatchingHelper.stripYear(cNorm); - final tBase = PhoneticMatchingHelper.stripYear(tNorm); - if (cBase == tBase && cBase.isNotEmpty) return true; - - final cCore = PhoneticMatchingHelper.collapseSpaces(PhoneticMatchingHelper.stripDescriptors(cNorm)); - final tCore = PhoneticMatchingHelper.collapseSpaces(PhoneticMatchingHelper.stripDescriptors(tNorm)); - if (cCore == tCore && cCore.isNotEmpty) return true; - - return cNorm.contains(tBase) || tNorm.contains(cBase) || (tCore.isNotEmpty && cCore.contains(tCore)) || (cCore.isNotEmpty && tCore.contains(cCore)); -} diff --git a/test/eval/canonical_person_and_role_correctness_test.dart b/test/eval/canonical_person_and_role_correctness_test.dart deleted file mode 100644 index 3dd4d94..0000000 --- a/test/eval/canonical_person_and_role_correctness_test.dart +++ /dev/null @@ -1,311 +0,0 @@ -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:flutter_test/flutter_test.dart'; -import 'package:ai_rally_search/models/search_intent.dart'; -import 'package:ai_rally_search/models/search_query.dart'; -import 'package:ai_rally_search/models/search_results.dart'; -import 'package:ai_rally_search/models/result_referent_context.dart'; -import 'package:ai_rally_search/services/database_service.dart'; -import 'package:ai_rally_search/services/search_repository.dart'; -import 'package:ai_rally_search/services/llm/llm_provider_config.dart'; -import 'package:ai_rally_search/services/llm/llm_query_parser.dart'; -import 'package:ai_rally_search/services/llm/query_understanding_spec.dart'; -import 'package:ai_rally_search/services/llm/query_output_validator.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/database_entity_resolver.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/entity_lookup_repository.dart'; - -void main() { - late DatabaseService dbService; - late SearchRepository searchRepo; - late DatabaseEntityLookupRepository lookupRepo; - late DatabaseEntityResolver resolver; - - setUpAll(() async { - await dotenv.load(fileName: '.env'); - dbService = DatabaseService(); - searchRepo = SearchRepository(dbService: dbService); - lookupRepo = DatabaseEntityLookupRepository(dbService: dbService); - resolver = DatabaseEntityResolver(repository: lookupRepo); - }); - - group('1. LLM QueryUnderstandingSpec & QueryOutputValidator personRole Tests', () { - test('QueryUnderstandingSpec contains personRole in all provider schemas', () { - // OpenAI JSON Schema - final openAiProps = (QueryUnderstandingSpec.jsonSchema['schema'] as Map)['properties'] as Map; - expect(openAiProps.containsKey('personRole'), isTrue); - final openAiPersonRole = openAiProps['personRole'] as Map; - expect(openAiPersonRole['enum'], containsAll(['ANY', 'DRIVER', 'CO_DRIVER'])); - - // Gemini Response Schema - final geminiProps = QueryUnderstandingSpec.geminiResponseSchema['properties'] as Map; - expect(geminiProps.containsKey('personRole'), isTrue); - final geminiPersonRole = geminiProps['personRole'] as Map; - expect(geminiPersonRole['enum'], containsAll(['ANY', 'DRIVER', 'CO_DRIVER'])); - - // System prompt instructions - expect(QueryUnderstandingSpec.systemPrompt, contains('personRole')); - expect(QueryUnderstandingSpec.systemPrompt, contains('CO_DRIVER')); - expect(QueryUnderstandingSpec.systemPrompt, contains('DRIVER')); - }); - - test('QueryOutputValidator parses personRole for all roles', () { - // DRIVER - final driverMap = { - 'intent': 'SEARCH_DRIVER_RALLIES', - 'driverNames': ['Josh Moffett'], - 'personRole': 'DRIVER', - }; - final resDriver = QueryOutputValidator.validateMap(jsonMap: driverMap); - expect(resDriver.isSuccess, isTrue); - expect(resDriver.query!.personRole, PersonRole.driver); - - // CO_DRIVER - final codriverMap = { - 'intent': 'SEARCH_DRIVER_RALLIES', - 'driverNames': ['Max Freeman'], - 'personRole': 'CO_DRIVER', - }; - final resCodriver = QueryOutputValidator.validateMap(jsonMap: codriverMap); - expect(resCodriver.isSuccess, isTrue); - expect(resCodriver.query!.personRole, PersonRole.coDriver); - - // ANY - final anyMap = { - 'intent': 'SEARCH_DRIVER_RALLIES', - 'driverNames': ['Max Freeman'], - 'personRole': 'ANY', - }; - final resAny = QueryOutputValidator.validateMap(jsonMap: anyMap); - expect(resAny.isSuccess, isTrue); - expect(resAny.query!.personRole, PersonRole.any); - - // Default / Missing -> ANY - final defaultMap = { - 'intent': 'SEARCH_DRIVER_RALLIES', - 'driverNames': ['Max Freeman'], - }; - final resDefault = QueryOutputValidator.validateMap(jsonMap: defaultMap); - expect(resDefault.isSuccess, isTrue); - expect(resDefault.query!.personRole, PersonRole.any); - }); - }); - - group('2. Max Freeman Golden Verification (Live DB Relational Truth)', () { - test('Max Freeman: DRIVER -> 0, CO_DRIVER -> 9, ANY -> 9', () async { - // Independent raw SQL truth queries - const codriverTruthSql = ''' - SELECT DISTINCT ev.event_id, ev.event_name - FROM rally_entry_list el - INNER JOIN user_codriver_profile cdp ON el.user_co_driver_id = cdp.codriver_id - INNER JOIN rally_sub_events se ON el.sub_event_id = se.sub_event_id - INNER JOIN rally_events ev ON se.event_id = ev.event_id - WHERE LOWER(cdp.full_name) LIKE '%max freeman%' - '''; - final rawCodriverRows = await dbService.query(codriverTruthSql); - final expectedCodriverEventIds = rawCodriverRows.map((r) => r['event_id'].toString()).toSet(); - expect(expectedCodriverEventIds.length, 9); - - const driverTruthSql = ''' - SELECT DISTINCT ev.event_id, ev.event_name - FROM rally_entry_list el - INNER JOIN user_driver_profile dp ON el.user_driver_id = dp.driver_id - INNER JOIN rally_sub_events se ON el.sub_event_id = se.sub_event_id - INNER JOIN rally_events ev ON se.event_id = ev.event_id - WHERE LOWER(dp.full_name) LIKE '%max freeman%' - '''; - final rawDriverRows = await dbService.query(driverTruthSql); - final expectedDriverEventIds = rawDriverRows.map((r) => r['event_id'].toString()).toSet(); - expect(expectedDriverEventIds.length, 0); - - // 1. Resolve and search with PersonRole.coDriver - final qCoDriver = SearchQuery( - intent: SearchIntent.searchDriverRallies, - driverNames: ['Max Freeman'], - personRole: PersonRole.coDriver, - ); - final resCoDriver = await resolver.resolve(qCoDriver); - final responseCoDriver = await searchRepo.search(resCoDriver.resolvedQuery!); - expect(responseCoDriver.totalCount, 9); - final actualCodriverIds = responseCoDriver.results.map((r) => r.rallyId).toSet(); - expect(actualCodriverIds, equals(expectedCodriverEventIds)); - - // 2. Resolve and search with PersonRole.driver - final qDriver = SearchQuery( - intent: SearchIntent.searchDriverRallies, - driverNames: ['Max Freeman'], - personRole: PersonRole.driver, - ); - final resDriver = await resolver.resolve(qDriver); - final responseDriver = await searchRepo.search(resDriver.resolvedQuery!); - expect(responseDriver.totalCount, 0); - expect(responseDriver.results.isEmpty, isTrue); - - // 3. Resolve and search with PersonRole.any - final qAny = SearchQuery( - intent: SearchIntent.searchDriverRallies, - driverNames: ['Max Freeman'], - personRole: PersonRole.any, - ); - final resAny = await resolver.resolve(qAny); - final responseAny = await searchRepo.search(resAny.resolvedQuery!); - expect(responseAny.totalCount, 9); - final actualAnyIds = responseAny.results.map((r) => r.rallyId).toSet(); - expect(actualAnyIds, equals(expectedCodriverEventIds)); - }); - }); - - group('3. Dual-Role Golden Verification (Live DB Account 419633b1-56ca-483a-8c10-0141d7cc3092)', () { - test('Chris Melly / Melly Chris: DRIVER -> 7, CO_DRIVER -> 16, ANY -> 23 distinct events', () async { - const accId = '419633b1-56ca-483a-8c10-0141d7cc3092'; - - // Independent DB truth queries - final driverTruthSql = ''' - SELECT DISTINCT ev.event_id, ev.event_name - FROM rally_entry_list el - INNER JOIN user_driver_profile dp ON el.user_driver_id = dp.driver_id - INNER JOIN rally_sub_events se ON el.sub_event_id = se.sub_event_id - INNER JOIN rally_events ev ON se.event_id = ev.event_id - WHERE dp.account_id = '$accId' - '''; - final rawDriverRows = await dbService.query(driverTruthSql); - final expectedDriverEventIds = rawDriverRows.map((r) => r['event_id'].toString()).toSet(); - expect(expectedDriverEventIds.length, 7); - - final codriverTruthSql = ''' - SELECT DISTINCT ev.event_id, ev.event_name - FROM rally_entry_list el - INNER JOIN user_codriver_profile cdp ON el.user_co_driver_id = cdp.codriver_id - INNER JOIN rally_sub_events se ON el.sub_event_id = se.sub_event_id - INNER JOIN rally_events ev ON se.event_id = ev.event_id - WHERE cdp.account_id = '$accId' - '''; - final rawCodriverRows = await dbService.query(codriverTruthSql); - final expectedCodriverEventIds = rawCodriverRows.map((r) => r['event_id'].toString()).toSet(); - expect(expectedCodriverEventIds.length, 16); - - final expectedAllEventIds = {...expectedDriverEventIds, ...expectedCodriverEventIds}; - expect(expectedAllEventIds.length, 23); - - // Verify EntityLookupRepository discovers BOTH roles and account_id bridge - final candidates = await lookupRepo.lookupDrivers('melly chris'); - expect(candidates.isNotEmpty, isTrue); - final cand = candidates.first; - expect(cand.metadata?['accountId'], accId); - expect(cand.metadata?['role'], 'both'); - expect(cand.metadata?['driverId'], isNotNull); - expect(cand.metadata?['codriverId'], isNotNull); - - // 1. DRIVER role query - final qDriver = SearchQuery( - intent: SearchIntent.searchDriverRallies, - driverNames: ['melly chris'], - personRole: PersonRole.driver, - ); - final resDriver = await resolver.resolve(qDriver); - final respDriver = await searchRepo.search(resDriver.resolvedQuery!); - expect(respDriver.totalCount, 7); - final actualDriverIds = respDriver.results.map((r) => r.rallyId).toSet(); - expect(actualDriverIds, equals(expectedDriverEventIds)); - - // 2. CO_DRIVER role query (even when queried using driver profile name "melly chris") - final qCodriver = SearchQuery( - intent: SearchIntent.searchDriverRallies, - driverNames: ['melly chris'], - personRole: PersonRole.coDriver, - ); - final resCodriver = await resolver.resolve(qCodriver); - final respCodriver = await searchRepo.search(resCodriver.resolvedQuery!); - expect(respCodriver.totalCount, 16); - final actualCodriverIds = respCodriver.results.map((r) => r.rallyId).toSet(); - expect(actualCodriverIds, equals(expectedCodriverEventIds)); - - // 3. ANY role query (unifies all 23 distinct events without duplication) - final qAny = SearchQuery( - intent: SearchIntent.searchDriverRallies, - driverNames: ['melly chris'], - personRole: PersonRole.any, - limit: 50, - ); - final resAny = await resolver.resolve(qAny); - final respAny = await searchRepo.search(resAny.resolvedQuery!); - expect(respAny.totalCount, 23); - final actualAnyIds = respAny.results.map((r) => r.rallyId).toSet(); - expect(actualAnyIds, equals(expectedAllEventIds)); - expect(actualAnyIds.length, 23); - }); - }); - - group('4. Conversational personRole Inheritance Tests', () { - test('Explicit role persists across multi-turn queries unless changed or cleared', () { - // Turn 1: "Which rallies did Max Freeman co-drive in?" - final turn1Query = SearchQuery( - intent: SearchIntent.searchDriverRallies, - driverNames: ['Max Freeman'], - personRole: PersonRole.coDriver, - ); - final turn1Resp = SearchResponse( - intent: SearchIntent.searchDriverRallies, - results: [], - totalCount: 9, - hasMore: false, - limit: 20, - offset: 0, - ); - final contextTurn1 = ResultReferentContext.fromSearchResponse( - turn1Resp, - queryDriver: 'Max Freeman', - queryPersonRole: turn1Query.personRole, - ); - expect(contextTurn1.activeDriver, 'Max Freeman'); - expect(contextTurn1.activePersonRole, PersonRole.coDriver); - - final searchContextTurn1 = SearchContext( - referents: contextTurn1, - previousQuery: turn1Query, - ); - final promptTextTurn1 = searchContextTurn1.formatPromptContext(); - expect(promptTextTurn1, contains('active driver is "Max Freeman" (role: CO_DRIVER)')); - expect(promptTextTurn1, contains('role: CO_DRIVER')); - - // Turn 2: Follow-up "What about 2026?" (inherits coDriver role) - final turn2Context = contextTurn1.copyWith(); - expect(turn2Context.activePersonRole, PersonRole.coDriver); - - // Turn 3: User changes role: "Now show rallies where he drove" - final turn3Context = turn2Context.copyWith(activePersonRole: PersonRole.driver); - expect(turn3Context.activePersonRole, PersonRole.driver); - - // Turn 4: User clears role: "Forget the role" - final turn4Context = turn3Context.copyWith(clearActivePersonRole: true); - expect(turn4Context.activePersonRole, isNull); - }); - }); - - group('5. Video & VideoAction Paths with Person Filters', () { - test('searchDriverVideos respects resolved person identity', () async { - final q = SearchQuery( - intent: SearchIntent.searchDriverVideos, - driverNames: ['Josh Moffett'], - limit: 10, - ); - final res = await resolver.resolve(q); - final resp = await searchRepo.search(res.resolvedQuery!); - expect(resp.intent, SearchIntent.searchDriverVideos); - expect(resp.results, isNotEmpty); - for (final vid in resp.results) { - expect(vid.driverName?.toLowerCase(), contains('moffett')); - } - }); - - test('searchVideoActions with person filter correctly filters highlights', () async { - final q = SearchQuery( - intent: SearchIntent.searchVideoActions, - driverNames: ['Josh Moffett'], - actionTypes: ['jump'], - limit: 10, - ); - final res = await resolver.resolve(q); - final resp = await searchRepo.search(res.resolvedQuery!); - expect(resp.intent, SearchIntent.searchVideoActions); - }); - }); -} diff --git a/test/eval/db_schema_inspector.dart b/test/eval/db_schema_inspector.dart deleted file mode 100644 index 953427e..0000000 --- a/test/eval/db_schema_inspector.dart +++ /dev/null @@ -1,42 +0,0 @@ -// ignore_for_file: avoid_print -import 'package:flutter_test/flutter_test.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:ai_rally_search/services/database_service.dart'; - -void main() { - TestWidgetsFlutterBinding.ensureInitialized(); - - test('Inspect rally_sub_events columns and Max Freeman entries', () async { - await dotenv.load(fileName: '.env'); - final db = DatabaseService(); - final conn = await db.connect(); - - final subEventsDesc = await conn.execute('DESCRIBE rally_sub_events;'); - print('rally_sub_events columns:'); - for (final r in subEventsDesc.rows) print(r.assoc()); - - // Search rally_entry_list for codriver_id = 7a633b52-950e-49ef-8cab-34cd43e99366 - final maxEntries = await conn.execute(''' - SELECT el.*, se.event_id, e.event_name, e.start_date, e.country - FROM rally_entry_list el - JOIN rally_sub_events se ON el.sub_event_id = se.sub_event_id - JOIN rally_events e ON se.event_id = e.event_id - WHERE el.user_co_driver_id = '7a633b52-950e-49ef-8cab-34cd43e99366'; - '''); - print('\nMax Freeman entries in rally_entry_list: (${maxEntries.rows.length})'); - for (final r in maxEntries.rows) print(r.assoc()); - - // Check if Max Freeman has any results in rally_results - final maxResults = await conn.execute(''' - SELECT rr.*, e.event_name - FROM rally_results rr - JOIN rally_entry_list el ON rr.entry_list_id = el.id - JOIN rally_events e ON rr.rally_id = e.event_id - WHERE el.user_co_driver_id = '7a633b52-950e-49ef-8cab-34cd43e99366'; - '''); - print('\nMax Freeman rows in rally_results: (${maxResults.rows.length})'); - for (final r in maxResults.rows) print(r.assoc()); - - await db.close(); - }); -} diff --git a/test/eval/debug_melly_sql_test.dart b/test/eval/debug_melly_sql_test.dart deleted file mode 100644 index 4c718f6..0000000 --- a/test/eval/debug_melly_sql_test.dart +++ /dev/null @@ -1,35 +0,0 @@ -// ignore_for_file: avoid_print -import 'package:flutter_test/flutter_test.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:ai_rally_search/services/database_service.dart'; - -void main() { - TestWidgetsFlutterBinding.ensureInitialized(); - - test('Inspect all dual-role accounts', () async { - await dotenv.load(fileName: '.env'); - final db = DatabaseService(); - - final allDual = await db.query(''' - SELECT - d.full_name AS driver_name, - cd.full_name AS codriver_name, - d.account_id, - d.driver_id, cd.codriver_id, - (SELECT COUNT(DISTINCT sub_event_id) FROM rally_entry_list WHERE user_driver_id = d.driver_id) AS driver_events, - (SELECT COUNT(DISTINCT sub_event_id) FROM rally_entry_list WHERE user_co_driver_id = cd.codriver_id) AS codriver_events - FROM user_driver_profile d - JOIN user_codriver_profile cd ON d.account_id = cd.account_id - WHERE d.driver_id IN (SELECT user_driver_id FROM rally_entry_list) - AND cd.codriver_id IN (SELECT user_co_driver_id FROM rally_entry_list) - LIMIT 20; - '''); - - print('Total dual-role accounts with entries in both: ${allDual.length}'); - for (final r in allDual) { - print(' Account ${r['account_id']}: Driver="${r['driver_name']}" (${r['driver_events']} ev) | Co-Driver="${r['codriver_name']}" (${r['codriver_events']} ev)'); - } - - await db.close(); - }); -} diff --git a/test/eval/deep_correctness_audit_test.dart b/test/eval/deep_correctness_audit_test.dart deleted file mode 100644 index db0b17b..0000000 --- a/test/eval/deep_correctness_audit_test.dart +++ /dev/null @@ -1,182 +0,0 @@ -// ignore_for_file: avoid_print -import 'package:flutter_test/flutter_test.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; - -import 'package:ai_rally_search/models/search_intent.dart'; -import 'package:ai_rally_search/models/search_query.dart'; -import 'package:ai_rally_search/services/database_service.dart'; -import 'package:ai_rally_search/services/search_repository.dart'; - -void main() { - TestWidgetsFlutterBinding.ensureInitialized(); - - group('Deep Search Result Correctness & LLM Spec Audit', () { - late DatabaseService db; - late SearchRepository repo; - - setUpAll(() async { - await dotenv.load(fileName: '.env'); - db = DatabaseService(); - repo = SearchRepository(dbService: db); - }); - - tearDownAll(() async { - await db.close(); - }); - - test('3B. Dual-Role Deep Audit', () async { - print('\n================================================================'); - print('SECTION 3B: DUAL-ROLE PERSON AUDIT'); - print('================================================================'); - - final dualRows = await db.query(''' - SELECT - d.full_name, - d.driver_id, cd.codriver_id, - (SELECT COUNT(DISTINCT sub_event_id) FROM rally_entry_list WHERE user_driver_id = d.driver_id) AS driver_events, - (SELECT COUNT(DISTINCT sub_event_id) FROM rally_entry_list WHERE user_co_driver_id = cd.codriver_id) AS codriver_events - FROM user_driver_profile d - JOIN user_codriver_profile cd ON d.account_id = cd.account_id - WHERE d.driver_id IN (SELECT user_driver_id FROM rally_entry_list) - AND cd.codriver_id IN (SELECT user_co_driver_id FROM rally_entry_list) - LIMIT 2; - '''); - - for (final person in dualRows) { - final name = person['full_name']?.toString() ?? ''; - final driverId = person['driver_id']; - final codriverId = person['codriver_id']; - - print('\nAuditing Dual-Role Person: "$name"'); - print(' Driver ID: $driverId (Events: ${person['driver_events']})'); - print(' Co-Driver ID: $codriverId (Events: ${person['codriver_events']})'); - - final rawDriverEvents = await db.query(''' - SELECT DISTINCT re.event_id, re.event_name, YEAR(re.start_date) as yr, 'driver' as role - FROM rally_entry_list el - JOIN rally_sub_events rse ON el.sub_event_id = rse.sub_event_id - JOIN rally_events re ON rse.event_id = re.event_id - WHERE el.user_driver_id = '$driverId' - '''); - - final rawCodriverEvents = await db.query(''' - SELECT DISTINCT re.event_id, re.event_name, YEAR(re.start_date) as yr, 'codriver' as role - FROM rally_entry_list el - JOIN rally_sub_events rse ON el.sub_event_id = rse.sub_event_id - JOIN rally_events re ON rse.event_id = re.event_id - WHERE el.user_co_driver_id = '$codriverId' - '''); - - final allEventMap = >{}; - for (final r in rawDriverEvents) { - allEventMap[r['event_id'].toString()] = r; - } - for (final r in rawCodriverEvents) { - allEventMap[r['event_id'].toString()] = r; - } - - print(' Raw Driver Distinct Events: ${rawDriverEvents.length}'); - print(' Raw Co-Driver Distinct Events: ${rawCodriverEvents.length}'); - print(' Raw Total Distinct Events: ${allEventMap.length}'); - - // Repository check - final repoAny = await repo.searchDriverRallies(SearchQuery( - intent: SearchIntent.searchDriverRallies, - driverName: name, - personRole: PersonRole.any, - )); - final repoDriver = await repo.searchDriverRallies(SearchQuery( - intent: SearchIntent.searchDriverRallies, - driverName: name, - personRole: PersonRole.driver, - )); - final repoCodriver = await repo.searchDriverRallies(SearchQuery( - intent: SearchIntent.searchDriverRallies, - driverName: name, - personRole: PersonRole.coDriver, - )); - - print(' Repo ANY Count: ${repoAny.totalCount}, Results: ${repoAny.results.length}'); - print(' Repo DRIVER Count: ${repoDriver.totalCount}, Results: ${repoDriver.results.length}'); - print(' Repo CODRIVER Count: ${repoCodriver.totalCount}, Results: ${repoCodriver.results.length}'); - - final expectedAnyIds = allEventMap.keys.toSet(); - final expectedDriverIds = rawDriverEvents.map((r) => r['event_id'].toString()).toSet(); - final expectedCodriverIds = rawCodriverEvents.map((r) => r['event_id'].toString()).toSet(); - - final actualAnyIds = repoAny.results.map((r) => r.rallyId).toSet(); - final actualDriverIds = repoDriver.results.map((r) => r.rallyId).toSet(); - final actualCodriverIds = repoCodriver.results.map((r) => r.rallyId).toSet(); - - print(' ANY Diff - Missing: ${expectedAnyIds.difference(actualAnyIds)}, Extra: ${actualAnyIds.difference(expectedAnyIds)}'); - print(' DRIVER Diff - Missing: ${expectedDriverIds.difference(actualDriverIds)}, Extra: ${actualDriverIds.difference(expectedDriverIds)}'); - print(' CODRIVER Diff - Missing: ${expectedCodriverIds.difference(actualCodriverIds)}, Extra: ${actualCodriverIds.difference(expectedCodriverIds)}'); - } - }); - - test('6. Counts, Pagination & Sub-Event Deduplication Audit', () async { - print('\n================================================================'); - print('SECTION 6: COUNTS, PAGINATION & SUB-EVENT DEDUPLICATION AUDIT'); - print('================================================================'); - - // Test event with multiple sub-events - final multiSubEvents = await db.query(''' - SELECT event_id, COUNT(sub_event_id) as sub_count - FROM rally_sub_events - GROUP BY event_id - HAVING sub_count > 1 - LIMIT 5; - '''); - print('Events with multiple sub-events:'); - for (final r in multiSubEvents) { - print(' Event ID: ${r['event_id']} (Sub-events: ${r['sub_count']})'); - } - - // Check count query vs results count for general rally search - final qRallies = SearchQuery(intent: SearchIntent.searchRallies, limit: 10, offset: 0); - final repoRallies = await repo.searchRallies(qRallies); - print('\nsearchRallies pagination:'); - print(' totalCount: ${repoRallies.totalCount}, returned: ${repoRallies.results.length}, hasMore: ${repoRallies.hasMore}'); - - // Check pagination page 2 - final qRalliesP2 = SearchQuery(intent: SearchIntent.searchRallies, limit: 10, offset: 10); - final repoRalliesP2 = await repo.searchRallies(qRalliesP2); - final page1Ids = repoRallies.results.map((r) => r.eventId).toSet(); - final page2Ids = repoRalliesP2.results.map((r) => r.eventId).toSet(); - final overlap = page1Ids.intersection(page2Ids); - print(' Page 1 & 2 overlap: $overlap (Should be empty)'); - }); - - test('7. Multi-Value Query Semantics Audit', () async { - print('\n================================================================'); - print('SECTION 7: MULTI-VALUE QUERY SEMANTICS AUDIT'); - print('================================================================'); - - // OR within dimension: driverNames - final multiDriverQ = SearchQuery( - intent: SearchIntent.searchDriverRallies, - driverNames: ['Max Freeman', 'Josh Moffett'], - driverMatchMode: MatchMode.any, - ); - final multiDriverRes = await repo.searchDriverRallies(multiDriverQ); - print('OR Drivers ("Max Freeman" OR "Josh Moffett"): Found ${multiDriverRes.totalCount} rallies'); - - // Multi-year within dimension - final multiYearQ = SearchQuery( - intent: SearchIntent.searchRallies, - years: [2025, 2026], - ); - final multiYearRes = await repo.searchRallies(multiYearQ); - print('OR Years (2025 OR 2026): Found ${multiYearRes.totalCount} rallies'); - - // Cross-dimension: Country + Year - final crossQ = SearchQuery( - intent: SearchIntent.searchRallies, - countries: ['Ireland'], - years: [2025], - ); - final crossRes = await repo.searchRallies(crossQ); - print('AND Cross-dimension (Ireland AND 2025): Found ${crossRes.totalCount} rallies'); - }); - }); -} diff --git a/test/eval/entity_search/audited_resolver_safety_benchmark_test.dart b/test/eval/entity_search/audited_resolver_safety_benchmark_test.dart deleted file mode 100644 index 431c556..0000000 --- a/test/eval/entity_search/audited_resolver_safety_benchmark_test.dart +++ /dev/null @@ -1,511 +0,0 @@ -// ignore_for_file: avoid_print -@Tags(['live-db', 'benchmark']) -library; - -import 'dart:convert'; -import 'dart:io'; - -import 'package:ai_rally_search/models/entity_candidate.dart'; -import 'package:ai_rally_search/models/search_intent.dart'; -import 'package:ai_rally_search/models/search_query.dart'; -import 'package:ai_rally_search/services/database_service.dart'; -import 'package:ai_rally_search/services/entity_search/entity_search_lookup_adapter.dart'; -import 'package:ai_rally_search/services/entity_search/controlled_fallback_entity_resolver.dart'; -import 'package:ai_rally_search/services/entity_search/in_memory_entity_search_service.dart'; -import 'package:ai_rally_search/services/entity_search/mysql_entity_search_data_source.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/database_entity_resolver.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/entity_lookup_repository.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/phonetic_matching_helper.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:flutter_test/flutter_test.dart'; - -void main() { - test( - 'NEW retrieval plus existing resolver: audited 168-query safety suite', - () async { - await dotenv.load(fileName: '.env'); - final db = DatabaseService(); - final service = InMemoryEntitySearchService( - dataSource: MySqlEntitySearchDataSource(database: db), - ); - await service.rebuild(); - final old = DatabaseEntityLookupRepository(dbService: db); - final fallbackMetrics = EntitySearchFallbackMetrics(); - final newResolver = DatabaseEntityResolver( - repository: EntitySearchLookupAdapter( - searchService: service, - cityFallback: old, - metrics: fallbackMetrics, - ), - ); - final resolver = ControlledFallbackEntityResolver( - legacyResolver: DatabaseEntityResolver(repository: old), - entitySearchResolver: newResolver, - config: const EntitySearchFallbackConfig( - mode: EntitySearchFallbackMode.fallback, - ), - metrics: fallbackMetrics, - ); - final positives = _positives; - final negatives = _negatives; - expect(positives.length, 62); - expect(negatives.length, 106); - - // Export frozen 168 safety cases fixture - final safetyCases = >[]; - var safetyIdx = 1; - for (final p in positives) { - safetyCases.add({ - 'caseId': 'safety_${safetyIdx++}', - 'category': 'positive', - 'input': p.input, - 'expectedCanonicalName': p.canonical, - 'entityType': p.type.name, - 'personRole': p.type == EntityType.driver ? 'driver' : null, - }); - } - for (final n in negatives) { - safetyCases.add({ - 'caseId': 'safety_${safetyIdx++}', - 'category': 'negative_confusable', - 'input': n.input, - 'expectedCanonicalName': null, - 'entityType': n.type.name, - 'personRole': n.type == EntityType.driver ? 'driver' : null, - }); - } - File('test/eval/entity_search/frozen_168_safety_cases.json').writeAsStringSync( - const JsonEncoder.withIndent(' ').convert(safetyCases), - ); - - var correctConfident = 0, - wrongPositiveConfident = 0, - positiveClarification = 0, - positiveNoMatch = 0; - var negativeWrongConfident = 0, - negativeClarification = 0, - negativeRejection = 0; - final positiveDetails = >[]; - final negativeDetails = >[]; - - for (final item in positives) { - final result = await resolver.resolve(_query(item.input, item.type)); - final resolved = result.resolutions.values - .where((r) => r.isResolved) - .firstOrNull - ?.resolvedCandidate - ?.canonicalName; - final correct = - resolved != null && _sameTarget(resolved, item.canonical); - if (correct) { - correctConfident++; - } else if (resolved != null) { - wrongPositiveConfident++; - } else if (result.requiresClarification) { - positiveClarification++; - } else { - positiveNoMatch++; - } - positiveDetails.add({ - 'input': item.input, - 'canonical': item.canonical, - 'type': item.type.name, - 'resolved': resolved, - 'correct': correct, - 'clarification': result.requiresClarification, - 'error': result.error, - }); - } - for (final item in negatives) { - final result = await resolver.resolve(_query(item.input, item.type)); - final resolved = result.resolutions.values - .where((r) => r.isResolved) - .firstOrNull - ?.resolvedCandidate - ?.canonicalName; - if (resolved != null) { - negativeWrongConfident++; - } else if (result.requiresClarification) { - negativeClarification++; - } else { - negativeRejection++; - } - negativeDetails.add({ - 'input': item.input, - 'type': item.type.name, - 'resolved': resolved, - 'clarification': result.requiresClarification, - 'error': result.error, - }); - } - final nullAccountSafety = >[]; - for (final item in const <(String, PersonRole, String)>[ - ('Shea Breen', PersonRole.any, 'same visible name / different IDs'), - ('Paweł Molgo', PersonRole.coDriver, 'wrong PersonRole'), - ('David Young', PersonRole.any, 'common surname collision'), - ('Paweł', PersonRole.any, 'partial person name'), - ('John O\'Sullivan', PersonRole.driver, 'duplicate person names'), - ]) { - final result = await resolver.resolve( - SearchQuery( - intent: SearchIntent.searchDriverRallies, - driverNames: [item.$1], - personRole: item.$2, - ), - ); - final resolved = result.resolutions.values - .where((resolution) => resolution.isResolved) - .firstOrNull - ?.resolvedCandidate; - nullAccountSafety.add({ - 'case': item.$3, - 'input': item.$1, - 'role': item.$2.name, - 'resolvedCanonicalId': resolved?.id, - 'clarification': result.requiresClarification, - 'error': result.error, - 'safe': resolved == null, - }); - } - final report = { - 'totalQueries': positives.length + negatives.length, - 'positive': { - 'queries': positives.length, - 'correctConfident': correctConfident, - 'wrongConfident': wrongPositiveConfident, - 'clarification': positiveClarification, - 'noMatch': positiveNoMatch, - 'details': positiveDetails, - }, - 'negativeConfusable': { - 'queries': negatives.length, - 'wrongConfident': negativeWrongConfident, - 'clarification': negativeClarification, - 'rejection': negativeRejection, - 'details': negativeDetails, - }, - 'falseConfidentAutoResolution': - wrongPositiveConfident + negativeWrongConfident, - 'fallbackTelemetry': fallbackMetrics.toMap(), - 'nullAccountSafety': nullAccountSafety, - }; - const path = - 'test/eval/entity_search/audited_resolver_safety_report.json'; - await File(path) - .writeAsString(const JsonEncoder.withIndent(' ').convert(report)); - expect(nullAccountSafety.every((entry) => entry['safe'] == true), isTrue); - print( - const JsonEncoder.withIndent(' ').convert({ - 'totalQueries': report['totalQueries'], - 'positive': { - for (final e in (report['positive'] as Map).entries) - if (e.key != 'details') e.key: e.value, - }, - 'negativeConfusable': { - for (final e in (report['negativeConfusable'] as Map).entries) - if (e.key != 'details') e.key: e.value, - }, - 'falseConfidentAutoResolution': - report['falseConfidentAutoResolution'], - }), - ); - await db.close(); - }, - timeout: const Timeout(Duration(minutes: 10)), - ); -} - -typedef _Case = ({String canonical, String input, EntityType type}); -typedef _Negative = ({String input, EntityType type}); - -SearchQuery _query(String input, EntityType type) => SearchQuery( - intent: type == EntityType.driver - ? SearchIntent.searchDriverVideos - : type == EntityType.rally - ? SearchIntent.searchRallies - : SearchIntent.searchVideoActions, - driverNames: type == EntityType.driver ? [input] : const [], - rallyNames: type == EntityType.rally ? [input] : const [], - stageNames: type == EntityType.stage ? [input] : const [], - personRole: PersonRole.any, -); - -bool _sameTarget(String actual, String expected) { - final a = PhoneticMatchingHelper.collapseSpaces( - PhoneticMatchingHelper.stripDescriptors(actual), - ); - final e = PhoneticMatchingHelper.collapseSpaces( - PhoneticMatchingHelper.stripDescriptors(expected), - ); - return a == e || a.contains(e) || e.contains(a); -} - -const _positives = <_Case>[ - (canonical: 'Rally Alūksne 2026', input: 'aluksnay', type: EntityType.rally), - ( - canonical: 'Rally Alūksne 2026', - input: 'a looks nay', - type: EntityType.rally, - ), - (canonical: 'Rally Alūksne 2026', input: 'alux new', type: EntityType.rally), - (canonical: 'Rally Alūksne 2026', input: 'eluksne', type: EntityType.rally), - (canonical: 'Rally Alūksne 2026', input: 'aluknse', type: EntityType.rally), - (canonical: 'Rally Alūksne 2026', input: 'aluksney', type: EntityType.rally), - (canonical: 'Paweł Molgo', input: 'pawel malgo', type: EntityType.driver), - (canonical: 'Shea Breen', input: 'shea brain', type: EntityType.driver), - ( - canonical: 'Donegal International Rally', - input: 'donny gall rally', - type: EntityType.rally, - ), - ( - canonical: 'Woodstoxx Kemmelberg 1', - input: 'kemel berg', - type: EntityType.stage, - ), - ( - canonical: 'Duszniki - Zieleniec 2', - input: 'dushniki', - type: EntityType.stage, - ), - ( - canonical: '6 Uren van Kortrijk 2024', - input: 'kortrik', - type: EntityType.rally, - ), - ( - canonical: 'Rali Serras de Fafe 2025', - input: 'Serras de Fafe', - type: EntityType.rally, - ), - ( - canonical: '7bet Rally Lazdijai 2025', - input: 'lazdiai', - type: EntityType.rally, - ), - ( - canonical: "Rali Terras d'Aboboreira 2026", - input: 'aboborera', - type: EntityType.rally, - ), - ( - canonical: 'Polski Rajd Legend 2026', - input: 'Polski Raid Legend', - type: EntityType.rally, - ), - ( - canonical: 'Rally Vranov 2026', - input: 'Rally Vranow', - type: EntityType.rally, - ), - ( - canonical: 'OBM Land der 1000 Hügel Rallye 2026', - input: '1000 Hugel Rallye', - type: EntityType.rally, - ), - ( - canonical: 'Rallijsprints Cesavine 2026', - input: 'Cesavine', - type: EntityType.rally, - ), - ( - canonical: 'Rallye Régional des Ardennes 2025', - input: 'Regional des Ardennes', - type: EntityType.rally, - ), - ( - canonical: 'Century 21 Portugal Rally Series - Castelo Branco 2025', - input: 'Castelo Branco 2025', - type: EntityType.rally, - ), - ( - canonical: 'Assess Ireland International Rally of the Lakes 2026', - input: 'Rally of the Lakes', - type: EntityType.rally, - ), - ( - canonical: 'Clonakilty Park Hotel West Cork Rally 2026', - input: 'West Cork Rally', - type: EntityType.rally, - ), - ( - canonical: 'Samsonas Rally Fivemiletown 2026', - input: 'Fivemiletown Rally', - type: EntityType.rally, - ), - ( - canonical: 'Modern Tyres Ulster Rally 2025', - input: 'Ulster Rally 2025', - type: EntityType.rally, - ), - ( - canonical: "Raven's Rock Stages Rally 2025", - input: 'Ravens Rock Stages', - type: EntityType.rally, - ), - ( - canonical: 'Birr Stages Rally 2026', - input: 'Birr Stages 2026', - type: EntityType.rally, - ), - ( - canonical: 'Fastnet Stages Rally 2025', - input: 'Fastnet Stages 2025', - type: EntityType.rally, - ), - ( - canonical: 'HK Cavan Stages Rally 2025', - input: 'Cavan Stages 2025', - type: EntityType.rally, - ), - ( - canonical: 'Jon-Gunnar Støten', - input: 'Jon Gunnar Stoten', - type: EntityType.driver, - ), - ( - canonical: 'Michal Babička', - input: 'Michal Babicka', - type: EntityType.driver, - ), - (canonical: 'Adam Zelík', input: 'Adam Zelik', type: EntityType.driver), - ( - canonical: 'Věroslav Cvrček', - input: 'Veroslav Cvrcek', - type: EntityType.driver, - ), - ( - canonical: 'Piotr Krotoszyński', - input: 'Piotr Krotoszynski', - type: EntityType.driver, - ), - (canonical: 'Hervé Emeriau', input: 'Herve Emerio', type: EntityType.driver), - (canonical: 'José Paula', input: 'Jose Pawla', type: EntityType.driver), - ( - canonical: 'Sergio Ramón Arrom', - input: 'Sergio Ramon', - type: EntityType.driver, - ), - ( - canonical: 'Raphaël Czwartkowski', - input: 'Raphael Czwartkovski', - type: EntityType.driver, - ), - (canonical: 'Vítor Matias', input: 'Vitor Mathias', type: EntityType.driver), - ( - canonical: "Stephen O'Connor", - input: 'Steven OConnor', - type: EntityType.driver, - ), - ( - canonical: "Diarmuid O'Toole", - input: 'Dermot OToole', - type: EntityType.driver, - ), - ( - canonical: 'Tanja Zingelmann-Hartjen', - input: 'Tanja Zingelmann', - type: EntityType.driver, - ), - ( - canonical: 'Nenad Lončarič', - input: 'Nenad Loncarich', - type: EntityType.driver, - ), - ( - canonical: 'Matej Bogović', - input: 'Matej Bogovich', - type: EntityType.driver, - ), - (canonical: 'Andrej Medić', input: 'Andrej Medich', type: EntityType.driver), - ( - canonical: 'John Shanahan jnr.', - input: 'John Shanahan Jr', - type: EntityType.driver, - ), - (canonical: 'Max Freeman', input: 'Max Frieman', type: EntityType.driver), - (canonical: 'Jan-Erik Mäll', input: 'Jan Erik Mall', type: EntityType.driver), - ( - canonical: 'Catharina Schmidt', - input: 'Katarina Schmidt', - type: EntityType.driver, - ), - (canonical: 'Paweł Molgo', input: 'Pawel Molgo', type: EntityType.driver), - (canonical: 'Shea Breen', input: 'Shea Breen', type: EntityType.driver), - (canonical: 'Jon-Gunnar Støten', input: 'Stoten', type: EntityType.driver), - (canonical: 'Věroslav Cvrček', input: 'Cvrcek', type: EntityType.driver), - ( - canonical: 'Woodstoxx Kemmelberg 1', - input: 'Kemmelberg 1', - type: EntityType.stage, - ), - ( - canonical: 'Duszniki - Zieleniec 2', - input: 'Duszniki Zieleniec', - type: EntityType.stage, - ), - (canonical: 'Seixoso 2', input: 'Seiksozo', type: EntityType.stage), - (canonical: 'Drumhallagh 2', input: 'Drumhalagh', type: EntityType.stage), - (canonical: 'Dikkebus 1', input: 'Dikebus', type: EntityType.stage), - ( - canonical: 'Fafe 2Powerstage', - input: 'Fafe Powerstage', - type: EntityType.stage, - ), - (canonical: 'Knockalla 2', input: 'Knokalla', type: EntityType.stage), - (canonical: 'Dunworley 2', input: 'Dunworly', type: EntityType.stage), - (canonical: 'Kellymount 1', input: 'Kelley Mount 1', type: EntityType.stage), -]; - -final _negatives = <_Negative>[ - for (final name in [ - 'Josh Smith', - 'Sam Williams', - "Keith O'Connor", - 'Craig McErlean', - 'Callum Breen', - 'Paul Moffett', - 'David Cronin', - 'Michael Devine', - 'Mark Freeman', - 'John Breen', - 'Brain', - 'Breenan', - 'Moffitt', - 'Moffat', - 'Cronan', - 'Devaney', - 'Molgow', - 'Stotenberg', - 'Zelinski', - 'Babic', - ]) - (input: name, type: EntityType.driver), - for (final name in [ - 'Rally of the Mountains', - 'International Stages', - 'West Coast Rally', - 'Cork 25 Stages', - 'Donegal 1972', - 'Galway 1981', - 'Lakes Rally 1990', - 'Ulster Stages 1965', - 'Aluksne 1999', - 'Fafe Classic 1985', - ]) - (input: name, type: EntityType.rally), - for (final name in [ - 'Super Stage 1', - 'Powerstage Final', - 'Mountain Pass 2', - 'Forest Stage 3', - 'Sprint Stage 1', - 'Town Stage 2', - ]) - (input: name, type: EntityType.stage), - for (var i = 1; i <= 70; i++) - ( - input: 'FictionalEntity$i PseudoName', - type: i.isEven ? EntityType.driver : EntityType.rally, - ), -]; diff --git a/test/eval/entity_search/export_benchmarks_test.dart b/test/eval/entity_search/export_benchmarks_test.dart deleted file mode 100644 index 985df7b..0000000 --- a/test/eval/entity_search/export_benchmarks_test.dart +++ /dev/null @@ -1,126 +0,0 @@ -import 'dart:convert'; -import 'dart:io'; -import 'dart:math'; - -import 'package:ai_rally_search/services/database_service.dart'; -import 'package:ai_rally_search/services/entity_search/entity_search_models.dart'; -import 'package:ai_rally_search/services/entity_search/mysql_entity_search_data_source.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/phonetic_matching_helper.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:flutter_test/flutter_test.dart'; - -import 'deterministic_corruption_generator.dart'; -import 'held_out_entity_fixture.dart'; - -void main() { - test('export frozen benchmark cases', () async { - await dotenv.load(fileName: '.env'); - final database = DatabaseService(); - final source = MySqlEntitySearchDataSource(database: database); - final all = await source.loadEntities(); - await database.close(); - - // 1. Export 803 cases - final selected80 = []; - final fixtureSeedIdByCanonicalId = {}; - for (final entry in heldOutEntityIds.entries) { - for (final id in entry.value) { - final matches = all - .where( - (e) => - e.entityType == entry.key && - (e.canonicalId == id || - (entry.key == SearchEntityType.person && - e.metadata['accountId']?.toString() == id)), - ) - .toList(); - if (matches.isNotEmpty) { - selected80.add(matches.single); - fixtureSeedIdByCanonicalId[matches.single.canonicalId] = id; - } - } - } - - final generator803 = DeterministicCorruptionGenerator(heldOutSeed); - final cases803 = >[]; - for (final target in selected80) { - final corruptions = generator803.generate( - target.canonicalName, - fixtureSeedIdByCanonicalId[target.canonicalId] ?? target.canonicalId, - person: target.entityType == SearchEntityType.person, - ); - for (final c in corruptions) { - cases803.add({ - 'targetCanonicalId': target.canonicalId, - 'targetCanonicalName': target.canonicalName, - 'entityType': target.entityType.name, - 'corruptionKind': c.kind, - 'difficulty': c.difficulty.name, - 'input': c.value, - }); - } - } - File('test/eval/entity_search/frozen_803_cases.json').writeAsStringSync( - const JsonEncoder.withIndent(' ').convert(cases803), - ); - print('Exported ${cases803.length} frozen 803 cases'); - - // 2. Export 1108 person cases - final people = all.where((e) => e.entityType == SearchEntityType.person).toList(); - final groups = >{ - 'ACCOUNT_BACKED': people.where((e) => e.metadata['identityKind'] == 'account').toList(), - 'NULL_DRIVER': people.where((e) => e.metadata['identityKind'] == 'driver').toList(), - 'NULL_CODRIVER': people.where((e) => e.metadata['identityKind'] == 'codriver').toList(), - }; - final desired = {'ACCOUNT_BACKED': 20, 'NULL_DRIVER': 40, 'NULL_CODRIVER': 40}; - final selectedPerson = >{}; - const excludedNames = {'pawel molgo', 'shea breen', 'max freeman', 'chris melly', 'melly'}; - const personSeed = 20260828; - - final nameCounts = {}; - for (final p in people) { - final norm = PhoneticMatchingHelper.normalize(p.canonicalName); - nameCounts[norm] = (nameCounts[norm] ?? 0) + 1; - } - - for (final entry in groups.entries) { - final eligible = entry.value.where((entity) { - final normalized = PhoneticMatchingHelper.normalize(entity.canonicalName); - if ((nameCounts[normalized] ?? 0) > 1) return false; - return !excludedNames.any((ex) => normalized == ex || normalized.contains(ex)); - }).toList() - ..sort((a, b) => a.canonicalId.compareTo(b.canonicalId)) - ..shuffle( - Random(personSeed + switch (entry.key) { 'ACCOUNT_BACKED' => 1, 'NULL_DRIVER' => 2, _ => 3 }), - ); - selectedPerson[entry.key] = eligible.take(desired[entry.key]!).toList(); - } - - final generatorPerson = DeterministicCorruptionGenerator(personSeed); - final cases1108 = >[]; - for (final entry in selectedPerson.entries) { - for (final target in entry.value) { - final corruptions = generatorPerson.generate( - target.canonicalName, - target.canonicalId, - person: true, - ); - for (final c in corruptions) { - cases1108.add({ - 'group': entry.key, - 'targetCanonicalId': target.canonicalId, - 'targetCanonicalName': target.canonicalName, - 'entityType': 'person', - 'corruptionKind': c.kind, - 'difficulty': c.difficulty.name, - 'input': c.value, - }); - } - } - } - File('test/eval/entity_search/frozen_1108_cases.json').writeAsStringSync( - const JsonEncoder.withIndent(' ').convert(cases1108), - ); - print('Exported ${cases1108.length} frozen 1108 cases'); - }); -} diff --git a/test/eval/entity_search/full_universe_person_benchmark_test.dart b/test/eval/entity_search/full_universe_person_benchmark_test.dart deleted file mode 100644 index 0907318..0000000 --- a/test/eval/entity_search/full_universe_person_benchmark_test.dart +++ /dev/null @@ -1,444 +0,0 @@ -// ignore_for_file: avoid_print -@Tags(['live-db', 'benchmark']) -library; - -import 'dart:convert'; -import 'dart:io'; -import 'dart:math'; - -import 'package:ai_rally_search/models/search_intent.dart'; -import 'package:ai_rally_search/models/search_query.dart'; -import 'package:ai_rally_search/services/database_service.dart'; -import 'package:ai_rally_search/services/entity_search/entity_search_lookup_adapter.dart'; -import 'package:ai_rally_search/services/entity_search/entity_candidate_generator.dart'; -import 'package:ai_rally_search/services/entity_search/entity_search_models.dart'; -import 'package:ai_rally_search/services/entity_search/in_memory_entity_search_service.dart'; -import 'package:ai_rally_search/services/entity_search/mysql_entity_search_data_source.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/database_entity_resolver.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/entity_lookup_repository.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/phonetic_matching_helper.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:flutter_test/flutter_test.dart'; - -import 'deterministic_corruption_generator.dart'; - -const _seed = 20260828; -const _excludedNames = { - 'pawel molgo', - 'shea breen', - 'max freeman', - 'chris melly', - 'melly', -}; - -void main() { - test('full live PERSON universe benchmark and identity safety', () async { - await dotenv.load(fileName: '.env'); - final db = DatabaseService(); - final rssBefore = ProcessInfo.currentRss; - final buildWatch = Stopwatch()..start(); - final entities = await MySqlEntitySearchDataSource(database: db) - .loadEntities(); - final service = InMemoryEntitySearchService.fromEntities(entities); - final fullScanService = InMemoryEntitySearchService.fromEntities( - entities, - candidateGenerator: FullScanCandidateGenerator(), - ); - buildWatch.stop(); - final rssAfter = ProcessInfo.currentRss; - final people = entities - .where((entity) => entity.entityType == SearchEntityType.person) - .toList(); - final groups = >{ - 'ACCOUNT_BACKED': people - .where((e) => e.metadata['identityKind'] == 'account') - .toList(), - 'NULL_DRIVER': people - .where((e) => e.metadata['identityKind'] == 'driver') - .toList(), - 'NULL_CODRIVER': people - .where((e) => e.metadata['identityKind'] == 'codriver') - .toList(), - }; - final desired = { - 'ACCOUNT_BACKED': 20, - 'NULL_DRIVER': 40, - 'NULL_CODRIVER': 40, - }; - final selected = >{}; - for (final entry in groups.entries) { - final eligible = - entry.value.where((entity) { - final normalized = PhoneticMatchingHelper.normalize( - entity.canonicalName, - ); - return !_excludedNames.any( - (excluded) => - normalized == excluded || normalized.contains(excluded), - ); - }).toList()..shuffle( - Random( - _seed + - switch (entry.key) { - 'ACCOUNT_BACKED' => 1, - 'NULL_DRIVER' => 2, - _ => 3, - }, - ), - ); - selected[entry.key] = eligible.take(desired[entry.key]!).toList(); - } - - final generator = DeterministicCorruptionGenerator(_seed); - final metrics = { - for (final key in [...groups.keys, 'ALL_PERSON']) key: _Metrics(), - }; - final fullScanMetrics = { - for (final key in [...groups.keys, 'ALL_PERSON']) key: _Metrics(), - }; - final latencies = []; - final generationLatencies = []; - final scoringLatencies = []; - final generatedPools = []; - final candidateRecall = { - 'pool': 0, - 'top25': 0, - 'top50': 0, - 'top100': 0, - 'top200': 0, - }; - var fullScanEscapes = 0; - final differences = >[]; - for (final entry in selected.entries) { - for (final target in entry.value) { - for (final corruption in generator.generate( - target.canonicalName, - target.canonicalId, - person: true, - )) { - final request = EntitySearchRequest( - rawMention: corruption.value, - entityType: SearchEntityType.person, - personRole: switch (entry.key) { - 'NULL_DRIVER' => PersonRole.driver, - 'NULL_CODRIVER' => PersonRole.coDriver, - _ => PersonRole.any, - }, - limit: 10, - ); - final generated = service.candidateGenerator.generate(request); - final candidates = await service.search(request); - final fullCandidates = await fullScanService.search(request); - final stats = service.lastQueryStats!; - latencies.add(stats.latency.inMicroseconds); - generationLatencies.add( - stats.candidateGenerationLatency.inMicroseconds, - ); - scoringLatencies.add(stats.scoringLatency.inMicroseconds); - generatedPools.add(stats.generatedCandidatePool); - if (stats.usedFullScanEscape) fullScanEscapes++; - final generatedRank = generated.preRankedCanonicalIds.indexOf( - target.canonicalId, - ); - if (generatedRank >= 0) { - candidateRecall['pool'] = candidateRecall['pool']! + 1; - if (generatedRank < 25) { - candidateRecall['top25'] = candidateRecall['top25']! + 1; - } - if (generatedRank < 50) { - candidateRecall['top50'] = candidateRecall['top50']! + 1; - } - if (generatedRank < 100) { - candidateRecall['top100'] = candidateRecall['top100']! + 1; - } - if (generatedRank < 200) { - candidateRecall['top200'] = candidateRecall['top200']! + 1; - } - } - final index = candidates.indexWhere( - (candidate) => candidate.canonicalId == target.canonicalId, - ); - final rank = index < 0 ? null : index + 1; - final fullIndex = fullCandidates.indexWhere( - (candidate) => candidate.canonicalId == target.canonicalId, - ); - final fullRank = fullIndex < 0 ? null : fullIndex + 1; - metrics[entry.key]!.add(rank); - metrics['ALL_PERSON']!.add(rank); - fullScanMetrics[entry.key]!.add(fullRank); - fullScanMetrics['ALL_PERSON']!.add(fullRank); - final indexedIds = candidates.map((c) => c.canonicalId).join('|'); - final fullIds = fullCandidates.map((c) => c.canonicalId).join('|'); - if (indexedIds != fullIds) { - differences.add({ - 'targetId': target.canonicalId, - 'input': corruption.value, - 'indexedRank': rank, - 'fullScanRank': fullRank, - }); - } - } - } - } - - final old = DatabaseEntityLookupRepository(dbService: db); - final adapter = EntitySearchLookupAdapter( - searchService: service, - cityFallback: old, - ); - final resolver = DatabaseEntityResolver(repository: adapter); - final pawel = await _namedResult( - service, - resolver, - 'pawel malgo', - PersonRole.driver, - ); - final shea = { - for (final role in PersonRole.values) - role.name: await _namedResult(service, resolver, 'shea brain', role), - }; - final collisions = await _collisionAudit(people, resolver); - final parity = await _rawParity(db, resolver, people); - latencies.sort(); - generationLatencies.sort(); - scoringLatencies.sort(); - generatedPools.sort(); - final queryCount = metrics['ALL_PERSON']!.count; - final report = { - 'seed': _seed, - 'sample': { - for (final entry in selected.entries) entry.key: entry.value.length, - }, - 'metrics': { - for (final entry in metrics.entries) entry.key: entry.value.toMap(), - }, - 'fullScanMetrics': { - for (final entry in fullScanMetrics.entries) - entry.key: entry.value.toMap(), - }, - 'candidateGeneration': { - 'pool': _distribution(generatedPools), - 'candidateRecall': { - for (final entry in candidateRecall.entries) - entry.key: entry.value / queryCount, - }, - 'fullScanEscapeInvocations': fullScanEscapes, - 'indexedVsFullScanDifferenceCount': differences.length, - 'differences': differences, - }, - 'pawelMolgo': pawel, - 'sheaBreen': shea, - 'sameNameCollisions': collisions, - 'rawDbParity': parity, - 'performance': { - 'totalIndexedEntities': service.indexStats?.entityCount, - 'personEntities': people.length, - 'buildMicroseconds': buildWatch.elapsedMicroseconds, - 'representationSizeEstimateBytes': service.indexStats?.estimatedBytes, - 'canonicalRepresentationBytes': - service.indexStats?.canonicalEstimatedBytes, - 'postingListBytes': service.indexStats?.postingListEstimatedBytes, - 'rssDeltaBytes': rssAfter - rssBefore, - 'personQueryMicroseconds': _distribution(latencies), - 'candidateGenerationMicroseconds': _distribution(generationLatencies), - 'scoringMicroseconds': _distribution(scoringLatencies), - }, - }; - const path = - 'test/eval/entity_search/full_universe_person_benchmark_report.json'; - await File(path) - .writeAsString(const JsonEncoder.withIndent(' ').convert(report)); - print(const JsonEncoder.withIndent(' ').convert(report)); - expect(groups['ACCOUNT_BACKED']!.length, greaterThanOrEqualTo(20)); - expect(groups['NULL_DRIVER']!.length, greaterThanOrEqualTo(40)); - expect(groups['NULL_CODRIVER']!.length, greaterThanOrEqualTo(40)); - expect( - parity.values.every((value) => value['matchesRawDb'] == true), - isTrue, - ); - await db.close(); - }, timeout: const Timeout(Duration(minutes: 30))); -} - -Future> _namedResult( - InMemoryEntitySearchService service, - DatabaseEntityResolver resolver, - String input, - PersonRole role, -) async { - final candidates = await service.search( - EntitySearchRequest( - rawMention: input, - entityType: SearchEntityType.person, - personRole: role, - limit: 10, - ), - ); - final result = await resolver.resolve(_query(input, role)); - return { - 'role': role.name, - 'finalBehavior': result.requiresClarification - ? 'clarification' - : result.resolutions.values.any((r) => r.isResolved) - ? 'resolved' - : 'no_match', - 'resolvedIds': result.resolvedQuery?.driverIds ?? const [], - 'candidates': candidates.take(5).map((candidate) { - return { - 'rank': candidates.indexOf(candidate) + 1, - 'canonicalId': candidate.canonicalId, - 'canonicalName': candidate.canonicalName, - 'role': candidate.metadata['role'], - 'driverId': candidate.metadata['driverId'], - 'codriverId': candidate.metadata['codriverId'], - 'tokenScore': candidate.signals.tokenScore, - 'ngramScore': candidate.signals.ngramScore, - 'lexicalScore': candidate.signals.lexicalScore, - 'phoneticScore': candidate.signals.phoneticScore, - 'finalScore': candidate.score, - }; - }).toList(), - }; -} - -Future> _collisionAudit( - List people, - DatabaseEntityResolver resolver, -) async { - final byName = >{}; - for (final person in people) { - byName - .putIfAbsent( - PhoneticMatchingHelper.normalize(person.canonicalName), - () => [], - ) - .add(person); - } - final collisions = - byName.entries.where((entry) => entry.value.length > 1).toList() - ..sort((a, b) => b.value.length.compareTo(a.value.length)); - final details = >[]; - for (final collision in collisions.take(20)) { - final behaviors = {}; - for (final role in PersonRole.values) { - final result = await resolver.resolve( - _query(collision.value.first.canonicalName, role), - ); - behaviors[role.name] = result.requiresClarification - ? 'clarification' - : result.resolutions.values.any((r) => r.isResolved) - ? 'resolved' - : 'no_match'; - } - details.add({ - 'name': collision.value.first.canonicalName, - 'identities': collision.value - .map((e) => {'id': e.canonicalId, 'role': e.metadata['role']}) - .toList(), - 'behavior': behaviors, - }); - } - return {'collisionGroups': collisions.length, 'audited': details}; -} - -Future>> _rawParity( - DatabaseService db, - DatabaseEntityResolver resolver, - List people, -) async { - final linked = await db.query(''' - SELECT 'driver' AS role, dp.driver_id AS role_id - FROM user_driver_profile dp - WHERE dp.account_id IS NULL AND dp.full_name IS NOT NULL - AND EXISTS (SELECT 1 FROM rally_entry_list el WHERE el.user_driver_id = dp.driver_id) - UNION ALL - SELECT 'co_driver' AS role, cdp.codriver_id AS role_id - FROM user_codriver_profile cdp - WHERE cdp.account_id IS NULL AND cdp.full_name IS NOT NULL - AND EXISTS (SELECT 1 FROM rally_entry_list el WHERE el.user_co_driver_id = cdp.codriver_id); - '''); - final driverId = linked - .firstWhere((row) => row['role'] == 'driver')['role_id'] - .toString(); - final codriverId = linked - .firstWhere((row) => row['role'] == 'co_driver')['role_id'] - .toString(); - final driver = people.firstWhere( - (e) => e.metadata['driverId']?.toString() == driverId, - ); - final codriver = people.firstWhere( - (e) => e.metadata['codriverId']?.toString() == codriverId, - ); - final output = >{}; - for (final item in [ - (driver, PersonRole.driver), - (codriver, PersonRole.coDriver), - ]) { - final result = await resolver.resolve( - _query(item.$1.canonicalName, item.$2), - ); - final expectedId = item.$2 == PersonRole.driver - ? item.$1.metadata['driverId'].toString() - : item.$1.metadata['codriverId'].toString(); - final resolvedIds = result.resolvedQuery?.driverIds ?? const []; - final column = item.$2 == PersonRole.driver - ? 'user_driver_id' - : 'user_co_driver_id'; - final rows = await db.query( - 'SELECT id, sub_event_id FROM rally_entry_list WHERE $column = :id ORDER BY id LIMIT 5;', - {'id': expectedId}, - ); - output[item.$2.name] = { - 'canonicalId': item.$1.canonicalId, - 'expectedRoleId': expectedId, - 'resolvedQueryIds': resolvedIds, - 'rawEntryIds': rows.map((row) => row['id']?.toString()).toList(), - 'rawSubEventIds': rows - .map((row) => row['sub_event_id']?.toString()) - .toList(), - 'matchesRawDb': - resolvedIds.length == 1 && - resolvedIds.single == expectedId && - rows.isNotEmpty, - }; - } - return output; -} - -SearchQuery _query(String name, PersonRole role) => SearchQuery( - intent: SearchIntent.searchDriverRallies, - driverNames: [name], - personRole: role, -); - -class _Metrics { - int count = 0; - int at1 = 0; - int at5 = 0; - int at10 = 0; - double reciprocalRank = 0; - void add(int? rank) { - count++; - if (rank == null) return; - if (rank <= 1) at1++; - if (rank <= 5) at5++; - if (rank <= 10) at10++; - reciprocalRank += 1 / rank; - } - - Map toMap() => { - 'queries': count, - 'recallAt1': at1 / count, - 'recallAt5': at5 / count, - 'recallAt10': at10 / count, - 'mrr': reciprocalRank / count, - }; -} - -Map _distribution(List values) => { - 'count': values.length, - 'average': values.reduce((a, b) => a + b) / values.length, - 'p50': values[(values.length * .50).floor().clamp(0, values.length - 1)], - 'p95': values[(values.length * .95).floor().clamp(0, values.length - 1)], - 'max': values.last, -}; diff --git a/test/eval/entity_search/held_out_entity_search_benchmark_test.dart b/test/eval/entity_search/held_out_entity_search_benchmark_test.dart deleted file mode 100644 index 3231619..0000000 --- a/test/eval/entity_search/held_out_entity_search_benchmark_test.dart +++ /dev/null @@ -1,609 +0,0 @@ -// ignore_for_file: avoid_print -@Tags(['live-db', 'benchmark']) -library; - -import 'dart:convert'; -import 'dart:io'; -import 'dart:math'; - -import 'package:ai_rally_search/models/entity_candidate.dart'; -import 'package:ai_rally_search/models/search_intent.dart'; -import 'package:ai_rally_search/models/search_query.dart'; -import 'package:ai_rally_search/services/database_service.dart'; -import 'package:ai_rally_search/services/entity_search/entity_search_lookup_adapter.dart'; -import 'package:ai_rally_search/services/entity_search/entity_candidate_generator.dart'; -import 'package:ai_rally_search/services/entity_search/entity_search_models.dart'; -import 'package:ai_rally_search/services/entity_search/in_memory_entity_search_service.dart'; -import 'package:ai_rally_search/services/entity_search/mysql_entity_search_data_source.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/database_entity_resolver.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/entity_lookup_repository.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:flutter_test/flutter_test.dart'; - -import 'deterministic_corruption_generator.dart'; -import 'held_out_entity_fixture.dart'; - -void main() { - TestWidgetsFlutterBinding.ensureInitialized(); - - test('frozen OLD vs NEW held-out baseline', () async { - await dotenv.load(fileName: '.env'); - final database = DatabaseService(); - final source = MySqlEntitySearchDataSource(database: database); - final rssBefore = ProcessInfo.currentRss; - final databaseLoadWatch = Stopwatch()..start(); - final all = await source.loadEntities(); - databaseLoadWatch.stop(); - final service = InMemoryEntitySearchService.fromEntities(all); - final fullScanService = InMemoryEntitySearchService.fromEntities( - all, - candidateGenerator: FullScanCandidateGenerator(), - ); - final buildWatch = Stopwatch()..start(); - await service.rebuild(); - buildWatch.stop(); - final rssAfter = ProcessInfo.currentRss; - final old = DatabaseEntityLookupRepository(dbService: database); - final generator = DeterministicCorruptionGenerator(heldOutSeed); - final selected = []; - final fixtureSeedIdByCanonicalId = {}; - for (final entry in heldOutEntityIds.entries) { - for (final id in entry.value) { - final matches = all - .where( - (e) => - e.entityType == entry.key && - (e.canonicalId == id || - (entry.key == SearchEntityType.person && - e.metadata['accountId']?.toString() == id)), - ) - .toList(); - if (matches.isNotEmpty) { - selected.add(matches.single); - fixtureSeedIdByCanonicalId[matches.single.canonicalId] = id; - } - } - } - expect( - selected.length, - 80, - reason: 'The persisted live fixture must remain resolvable', - ); - - final metrics = {}; - final corruptionCounts = {}; - final personDiagnostics = >[]; - final pools = []; - final evaluated = []; - final returned = []; - final latencyByType = >{}; - final generationLatencyByType = >{}; - final scoringLatencyByType = >{}; - final generatedPoolsByType = >{}; - final fullScanEscapesByType = {}; - final candidateRecallByType = {}; - final overallCandidateRecall = _CandidateRecall(); - final indexedVsFullScanDifferences = >[]; - - for (final target in selected) { - final corruptions = generator.generate( - target.canonicalName, - fixtureSeedIdByCanonicalId[target.canonicalId] ?? target.canonicalId, - person: target.entityType == SearchEntityType.person, - ); - for (final corruption in corruptions) { - corruptionCounts.update( - '${corruption.kind}:${corruption.difficulty.name}', - (v) => v + 1, - ifAbsent: () => 1, - ); - final request = EntitySearchRequest( - rawMention: corruption.value, - entityType: target.entityType, - limit: 10, - ); - final generated = service.candidateGenerator.generate(request); - final generatedRank = generated.preRankedCanonicalIds.indexOf( - target.canonicalId, - ); - overallCandidateRecall.add(generatedRank); - candidateRecallByType - .putIfAbsent(target.entityType.name, _CandidateRecall.new) - .add(generatedRank); - final newCandidates = await service.search(request); - final fullScanCandidates = await fullScanService.search(request); - final stats = service.lastQueryStats!; - pools.add(stats.survivingCandidates); - evaluated.add(stats.rawCandidatesEvaluated); - returned.add(stats.returnedCandidates); - latencyByType - .putIfAbsent(target.entityType.name, () => []) - .add(stats.latency.inMicroseconds); - generationLatencyByType - .putIfAbsent(target.entityType.name, () => []) - .add(stats.candidateGenerationLatency.inMicroseconds); - scoringLatencyByType - .putIfAbsent(target.entityType.name, () => []) - .add(stats.scoringLatency.inMicroseconds); - generatedPoolsByType - .putIfAbsent(target.entityType.name, () => []) - .add(stats.generatedCandidatePool); - if (stats.usedFullScanEscape) { - fullScanEscapesByType.update( - target.entityType.name, - (value) => value + 1, - ifAbsent: () => 1, - ); - } - final oldCandidates = await _oldSearch( - old, - target.entityType, - corruption.value, - ); - final newRank = _newRank(newCandidates, target.canonicalId); - final fullScanRank = _newRank(fullScanCandidates, target.canonicalId); - final oldRank = _oldRank(oldCandidates, target); - for (final pair in [ - ('NEW', newRank), - ('FULL_SCAN', fullScanRank), - ('OLD', oldRank), - ]) { - _record(metrics, pair.$1, 'overall', pair.$2); - _record(metrics, pair.$1, 'type:${target.entityType.name}', pair.$2); - _record( - metrics, - pair.$1, - 'difficulty:${corruption.difficulty.name}', - pair.$2, - ); - } - final indexedIds = newCandidates.map((c) => c.canonicalId).toList(); - final fullIds = fullScanCandidates.map((c) => c.canonicalId).toList(); - if (indexedIds.join('|') != fullIds.join('|')) { - indexedVsFullScanDifferences.add({ - 'targetId': target.canonicalId, - 'input': corruption.value, - 'type': target.entityType.name, - 'indexedRank': newRank, - 'fullScanRank': fullScanRank, - 'indexedTop10': indexedIds, - 'fullScanTop10': fullIds, - }); - } - if (target.entityType == SearchEntityType.person) { - final found = newCandidates - .where((c) => c.canonicalId == target.canonicalId) - .firstOrNull; - personDiagnostics.add({ - 'accountId': target.canonicalId, - 'canonicalName': target.canonicalName, - 'driverId': target.metadata['driverId'], - 'codriverId': target.metadata['codriverId'], - 'role': target.metadata['role'], - 'corruptionKind': corruption.kind, - 'difficulty': corruption.difficulty.name, - 'corruption': corruption.value, - 'rank': newRank, - 'tokenScore': found?.signals.tokenScore, - 'ngramScore': found?.signals.ngramScore, - 'lexicalScore': found?.signals.lexicalScore, - 'phoneticScore': found?.signals.phoneticScore, - 'finalScore': found?.score, - 'top5': newCandidates - .take(5) - .map( - (c) => { - 'accountId': c.canonicalId, - 'name': c.canonicalName, - 'score': c.score, - }, - ) - .toList(), - }); - } - } - } - - final adapter = EntitySearchLookupAdapter( - searchService: service, - cityFallback: old, - ); - final resolver = DatabaseEntityResolver( - repository: adapter, - minConfidenceThreshold: 0.75, - minScoreGap: 0.15, - ); - final safety = await _runSafety(resolver, selected); - final personAudit = await _personNameAudit( - database, - heldOutEntityIds[SearchEntityType.person]!, - ); - pools.sort(); - evaluated.sort(); - returned.sort(); - - final report = { - 'generatedAt': DateTime.now().toUtc().toIso8601String(), - 'frozenImplementation': true, - 'seed': heldOutSeed, - 'heldOutComposition': { - for (final type in SearchEntityType.values) - type.name: selected.where((e) => e.entityType == type).length, - }, - 'heldOutEntities': selected - .map( - (e) => { - 'id': e.canonicalId, - 'name': e.canonicalName, - 'type': e.entityType.name, - 'metadata': e.metadata, - }, - ) - .toList(), - 'corruptionCounts': corruptionCounts, - 'retrievalMetrics': { - for (final entry in metrics.entries) entry.key: entry.value.toJson(), - }, - 'personDiagnostics': personDiagnostics, - 'canonicalPersonNamePolicy': { - 'currentPolicy': 'driver profile name when present, otherwise co-driver profile name; normalized lexical tie-break within a role', - 'audit': personAudit, - }, - 'resolverSafety': safety, - 'candidatePools': _distribution( - pools, - extras: { - 'meanRawEvaluated': _mean(evaluated), - 'meanReturned': _mean(returned), - 'maxRawEvaluated': evaluated.last, - }, - ), - 'candidateGeneration': { - 'fullScanEscapeInvocations': fullScanEscapesByType, - 'candidateRecall': { - 'overall': overallCandidateRecall.toJson(), - 'byType': { - for (final entry in candidateRecallByType.entries) - entry.key: entry.value.toJson(), - }, - }, - 'indexedVsFullScanDifferenceCount': indexedVsFullScanDifferences.length, - 'differences': indexedVsFullScanDifferences, - 'byType': { - for (final type in generatedPoolsByType.keys) - type: { - 'pool': _distribution(generatedPoolsByType[type]!..sort()), - 'generationLatencyMicroseconds': _distribution( - generationLatencyByType[type]!..sort(), - ), - 'scoringLatencyMicroseconds': _distribution( - scoringLatencyByType[type]!..sort(), - ), - }, - }, - }, - 'memory': { - 'representationSizeEstimateBytes': service.indexStats?.estimatedBytes, - 'canonicalRepresentationBytes': - service.indexStats?.canonicalEstimatedBytes, - 'postingListBytes': service.indexStats?.postingListEstimatedBytes, - 'processRssBeforeBytes': rssBefore, - 'processRssAfterBytes': rssAfter, - 'processRssDeltaBytes': rssAfter - rssBefore, - 'limitation': 'RSS includes DB loading/runtime allocation and is not an isolated Dart heap measurement.', - }, - 'performance': { - 'entityCount': service.indexStats?.entityCount, - 'databaseLoadMicroseconds': databaseLoadWatch.elapsedMicroseconds, - 'measuredRebuildMicroseconds': buildWatch.elapsedMicroseconds, - 'queryLatencyByTypeMicroseconds': { - for (final e in latencyByType.entries) - e.key: _distribution(e.value..sort()), - }, - }, - }; - const path = 'test/eval/entity_search/held_out_baseline_report.json'; - await File(path) - .writeAsString(const JsonEncoder.withIndent(' ').convert(report)); - print('WROTE $path'); - print( - const JsonEncoder.withIndent(' ').convert({ - 'composition': report['heldOutComposition'], - 'corruptions': corruptionCounts.values.fold(0, (a, b) => a + b), - 'metrics': report['retrievalMetrics'], - 'safety': safety, - 'pools': report['candidatePools'], - 'performance': report['performance'], - }), - ); - await database.close(); - }, timeout: const Timeout(Duration(minutes: 30))); -} - -Future> _oldSearch( - DatabaseEntityLookupRepository old, - SearchEntityType type, - String value, -) => switch (type) { - SearchEntityType.rally => old.lookupRallies(value, limit: 10), - SearchEntityType.person => old.lookupDrivers(value, limit: 10), - SearchEntityType.stage => old.lookupStages(value, limit: 10), - SearchEntityType.uploader => old.lookupUploaders(value, limit: 10), -}; - -int? _newRank(List values, String id) { - final index = values.indexWhere((c) => c.canonicalId == id); - return index < 0 ? null : index + 1; -} - -int? _oldRank(List values, CanonicalSearchEntity target) { - final index = values.indexWhere((c) { - if (target.entityType == SearchEntityType.person) { - return c.metadata?['accountId']?.toString() == target.canonicalId || - c.metadata?['driverId']?.toString() == - target.metadata['driverId']?.toString() || - c.metadata?['codriverId']?.toString() == - target.metadata['codriverId']?.toString(); - } - return c.id == target.canonicalId; - }); - return index < 0 ? null : index + 1; -} - -void _record( - Map metrics, - String engine, - String slice, - int? rank, -) => metrics.putIfAbsent('$engine:$slice', _Metrics.new).add(rank); - -class _Metrics { - int total = 0, r1 = 0, r5 = 0, r10 = 0; - double reciprocalRank = 0; - void add(int? rank) { - total++; - if (rank != null) { - if (rank <= 1) r1++; - if (rank <= 5) r5++; - if (rank <= 10) r10++; - reciprocalRank += 1 / rank; - } - } - - Map toJson() => { - 'cases': total, - 'recallAt1': r1 / total, - 'recallAt5': r5 / total, - 'recallAt10': r10 / total, - 'mrr': reciprocalRank / total, - }; -} - -class _CandidateRecall { - int total = 0; - int pool = 0; - int top25 = 0; - int top50 = 0; - int top100 = 0; - int top200 = 0; - - void add(int zeroBasedRank) { - total++; - if (zeroBasedRank < 0) return; - pool++; - if (zeroBasedRank < 25) top25++; - if (zeroBasedRank < 50) top50++; - if (zeroBasedRank < 100) top100++; - if (zeroBasedRank < 200) top200++; - } - - Map toJson() => { - 'cases': total, - 'pool': pool / total, - 'top25': top25 / total, - 'top50': top50 / total, - 'top100': top100 / total, - 'top200': top200 / total, - }; -} - -Map _distribution( - List sorted, { - Map extras = const {}, -}) => { - 'count': sorted.length, - 'mean': _mean(sorted), - 'p50': _percentile(sorted, .50), - 'p95': _percentile(sorted, .95), - 'max': sorted.last, - ...extras, -}; -double _mean(List values) => - values.fold(0, (a, b) => a + b) / values.length; -int _percentile(List values, double p) => - values[min(values.length - 1, (values.length * p).floor())]; - -Future>> _personNameAudit( - DatabaseService db, - List ids, -) async { - final quoted = ids.map((id) => "'${id.replaceAll("'", "''")}'").join(','); - final rows = await db.query(''' - SELECT account_id, 'driver' AS role, driver_id AS profile_id, full_name FROM user_driver_profile WHERE account_id IN ($quoted) - UNION ALL - SELECT account_id, 'co_driver' AS role, codriver_id AS profile_id, full_name FROM user_codriver_profile WHERE account_id IN ($quoted); - '''); - final grouped = >>{}; - for (final row in rows) { - grouped.putIfAbsent(row['account_id'].toString(), () => []).add(row); - } - return grouped.entries.map((e) { - final names = e.value - .map((r) => r['full_name']?.toString()) - .whereType() - .toSet(); - return { - 'accountId': e.key, - 'profiles': e.value, - 'distinctNames': names.toList(), - 'nameDivergence': names.length > 1, - }; - }).toList(); -} - -Future> _runSafety( - DatabaseEntityResolver resolver, - List entities, -) async { - final cases = <({String id, SearchQuery query, String? expected})>[ - ( - id: 'positive_aluksne', - query: const SearchQuery( - intent: SearchIntent.searchVideoActions, - rallyNames: ['alux new'], - ), - expected: 'Alūksne', - ), - ( - id: 'wrong_year', - query: const SearchQuery( - intent: SearchIntent.searchVideoActions, - rallyNames: ['Aluksne'], - years: [1999], - ), - expected: null, - ), - ( - id: 'wrong_person', - query: const SearchQuery( - intent: SearchIntent.searchDriverVideos, - driverNames: ['Craig Nonexistentperson'], - ), - expected: null, - ), - ( - id: 'nonsense_person', - query: const SearchQuery( - intent: SearchIntent.searchDriverVideos, - driverNames: ['Zzzz Qqqq Xxxx'], - ), - expected: null, - ), - ( - id: 'random_tourist_person', - query: const SearchQuery( - intent: SearchIntent.searchDriverVideos, - driverNames: ['Random Tourist 12345'], - ), - expected: null, - ), - ( - id: 'fake_rally', - query: const SearchQuery( - intent: SearchIntent.searchVideoActions, - rallyNames: ['Rally Fakeplacenamexyz'], - ), - expected: null, - ), - ( - id: 'random_city_rally', - query: const SearchQuery( - intent: SearchIntent.searchRallies, - rallyNames: ['Random City Nonexistent Stages Rally'], - ), - expected: null, - ), - ( - id: 'spaceship_rally', - query: const SearchQuery( - intent: SearchIntent.searchRallies, - rallyNames: ['Pineapple Spaceship Championship 2099'], - ), - expected: null, - ), - ( - id: 'moon_base_stage', - query: const SearchQuery( - intent: SearchIntent.searchVideoActions, - stageNames: ['Moon Base Alpha Stage 99'], - ), - expected: null, - ), - ( - id: 'coral_reef_stage', - query: const SearchQuery( - intent: SearchIntent.searchVideoActions, - stageNames: ['Underwater Coral Reef SS99'], - ), - expected: null, - ), - ( - id: 'unrelated_noise', - query: const SearchQuery( - intent: SearchIntent.searchDriverVideos, - driverNames: ['Completely Unrelated Noise'], - ), - expected: null, - ), - ( - id: 'common_first_wrong_surname', - query: const SearchQuery( - intent: SearchIntent.searchDriverVideos, - driverNames: ['James Xylophone'], - ), - expected: null, - ), - ]; - final onlyCoDriver = entities - .where( - (e) => - e.entityType == SearchEntityType.person && - e.metadata['role'] == 'co_driver', - ) - .firstOrNull; - if (onlyCoDriver != null) { - cases.add(( - id: 'driver_role_mismatch', - query: SearchQuery( - intent: SearchIntent.searchDriverVideos, - driverNames: [onlyCoDriver.canonicalName], - personRole: PersonRole.driver, - ), - expected: null, - )); - } - var falseConfident = 0, correct = 0, clarification = 0, noMatch = 0; - final details = >[]; - for (final item in cases) { - final result = await resolver.resolve(item.query); - final resolution = result.resolutions.values.firstOrNull; - final resolvedName = resolution?.resolvedCandidate?.canonicalName; - final auto = resolution?.isResolved ?? false; - final isCorrect = - item.expected != null && - auto && - (resolvedName?.contains(item.expected!) ?? false); - final isFalse = auto && !isCorrect; - if (isFalse) falseConfident++; - if (isCorrect) correct++; - if (result.requiresClarification) clarification++; - if (!auto && !result.requiresClarification) noMatch++; - details.add({ - 'id': item.id, - 'expected': item.expected, - 'resolved': resolvedName, - 'autoResolved': auto, - 'clarification': result.requiresClarification, - 'error': result.error, - 'falseConfident': isFalse, - }); - } - return { - 'cases': cases.length, - 'falseConfidentAutoResolution': falseConfident, - 'correctAutoResolution': correct, - 'clarification': clarification, - 'noMatch': noMatch, - 'details': details, - }; -} diff --git a/test/eval/entity_search/human_voice_corpus_benchmark_test.dart b/test/eval/entity_search/human_voice_corpus_benchmark_test.dart deleted file mode 100644 index 93eb9e8..0000000 --- a/test/eval/entity_search/human_voice_corpus_benchmark_test.dart +++ /dev/null @@ -1,612 +0,0 @@ -// ignore_for_file: avoid_print -@Tags(['live-db', 'live-api', 'benchmark']) -library; - -import 'dart:convert'; -import 'dart:io'; -import 'dart:math'; - -import 'package:ai_rally_search/services/database_service.dart'; -import 'package:ai_rally_search/services/entity_search/controlled_fallback_entity_resolver.dart'; -import 'package:ai_rally_search/services/entity_search/entity_search_lookup_adapter.dart'; -import 'package:ai_rally_search/services/entity_search/in_memory_entity_search_service.dart'; -import 'package:ai_rally_search/services/entity_search/mysql_entity_search_data_source.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/database_entity_resolver.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/entity_lookup_repository.dart'; -import 'package:ai_rally_search/services/llm/llm_query_parser_factory.dart'; -import 'package:ai_rally_search/services/speech/audio_preprocessor.dart'; -import 'package:ai_rally_search/services/speech/openai_speech_to_text_service.dart'; -import 'package:ai_rally_search/services/speech/speech_config.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:flutter_test/flutter_test.dart'; - -import 'human_voice_dynamic_top3_evaluator.dart'; -import 'human_voice_fixture_validator.dart'; -import 'human_voice_smoke_evaluator.dart'; - -void main() { - test('ES-8B validates and benchmarks the real-human corpus', () async { - await dotenv.load(fileName: '.env'); - final apiKey = dotenv.env['OPENAI_API_KEY']; - expect(apiKey, isNotNull); - expect(apiKey, isNotEmpty); - - final manifestFile = File( - 'test/eval/entity_search/human_voice_smoke_manifest.json', - ); - final manifest = - jsonDecode(await manifestFile.readAsString()) as Map; - final fixtures = (manifest['fixtures'] as List) - .cast>(); - expect(manifest['schemaVersion'], 'ES8B_HUMAN_FIXTURE_V1'); - expect(fixtures, isNotEmpty); - - final db = DatabaseService(); - final speech = OpenAiSpeechToTextService( - config: SpeechConfig( - providerType: SpeechProviderType.openAiDirectDev, - endpointUrl: 'https://api.openai.com/v1/audio/transcriptions', - apiKey: apiKey, - model: 'gpt-transcribe', - timeout: const Duration(seconds: 45), - ), - ); - try { - final entities = await MySqlEntitySearchDataSource(database: db) - .loadEntities(); - final validation = await const HumanVoiceFixtureValidator().validate( - manifest: manifest, - liveEntities: entities, - ); - expect( - validation['valid'], - isTrue, - reason: const JsonEncoder.withIndent(' ') - .convert(validation['issues']), - ); - expect(validation['fixturesSilentlyDropped'], 0); - - const noOp = NoOpAudioPreprocessor(); - for (final fixture in fixtures) { - final file = File(fixture['filePath'] as String); - final original = await file.readAsBytes(); - final processed = await noOp.process( - inputBytes: original, - filename: file.uri.pathSegments.last, - strategy: AudioPreprocessingStrategy.raw, - ); - expect(processed.changed, isFalse); - expect(processed.bytes, orderedEquals(original)); - expect(await file.readAsBytes(), orderedEquals(original)); - } - - final entitySearch = InMemoryEntitySearchService.fromEntities(entities); - final legacy = DatabaseEntityLookupRepository(dbService: db); - final resolver = ControlledFallbackEntityResolver( - legacyResolver: DatabaseEntityResolver(repository: legacy), - entitySearchResolver: DatabaseEntityResolver( - repository: EntitySearchLookupAdapter( - searchService: entitySearch, - cityFallback: legacy, - ), - ), - config: const EntitySearchFallbackConfig( - mode: EntitySearchFallbackMode.fallback, - ), - ); - final pipeline = HumanVoiceSmokeEvaluator( - speech: speech, - parser: LlmQueryParserFactory.create(), - resolver: resolver, - entitySearch: entitySearch, - ); - final evaluator = HumanVoiceDynamicTop3Evaluator(pipeline: pipeline); - final results = >[]; - for (final fixture in fixtures) { - results.add(await evaluator.evaluate(fixture: fixture)); - } - - final duplicateMembers = - ((validation['audioInventory'] as Map)['DUPLICATE_GROUPS'] as List) - .whereType() - .expand((group) { - final members = (group['members'] as List) - .whereType() - .toList(); - return members.skip(1); - }) - .toSet(); - final uniqueResults = results - .where((item) => !duplicateMembers.contains(item['recordingId'])) - .toList(growable: false); - final perFileMetrics = _metrics(results); - final uniqueAudioMetrics = _metrics(uniqueResults); - final wrongConfidentCases = _wrongConfidentCases(results); - final newlyIntroducedWrongConfident = wrongConfidentCases - .where((item) => item['newlyIntroducedByDynamic'] == true) - .toList(growable: false); - final triggeredRecoveries = results - .where((item) => item['secondPassTriggered'] == true) - .map(_triggerSummary) - .toList(growable: false); - final recommendation = _recommendation(uniqueAudioMetrics); - final permanentRegression = fixtures.firstWhere( - (fixture) => fixture['fixtureId'] == 'human-smoke-001', - )['permanentRegression']; - expect(permanentRegression, isA()); - expect( - (permanentRegression as Map)['regressionId'], - 'ES8A_ASIM1_DYNAMIC_TOP3_RECOVERY', - ); - expect(permanentRegression['frozenRawOutcome'], 'NO_MATCH'); - expect( - permanentRegression['dynamicTop3Outcome'], - 'CORRECT_CLARIFICATION', - ); - - final report = { - 'phase': 'ES-8B', - 'architectureStatus': 'FROZEN_FOR_HUMAN_DATA_COLLECTION', - 'humanBenchmarkStatus': 'LABELED_SMOKE_TEST_ONLY', - 'generatedAtUtc': DateTime.now().toUtc().toIso8601String(), - 'oneCommandWorkflow': 'flutter test test/eval/entity_search/human_voice_corpus_benchmark_test.dart --reporter expanded', - 'strategies': ['RAW_BASELINE', 'RAW_DYNAMIC_TOP3'], - 'productionVoiceRoutingChanged': false, - 'runtimeDefaultsChanged': false, - 'entitySearchRankingChanged': false, - 'resolverThresholdsChanged': false, - 'audioPreprocessing': { - 'implementation': 'NoOpAudioPreprocessor', - 'enabled': false, - }, - 'staticSttContextEnabled': false, - 'dynamicTop3ProductionEnabled': false, - 'top5OrTop10Tested': false, - 'manifest': { - 'path': manifestFile.path, - 'schemaVersion': manifest['schemaVersion'], - 'fixtureCount': fixtures.length, - }, - 'validation': validation, - 'corpusCoverage': validation['coverage'], - 'collectionMilestones': validation['collectionMilestones'], - 'entityCoverageGuidance': validation['entityCoverageGuidance'], - 'perFileMetrics': perFileMetrics, - 'uniqueAudioMetrics': uniqueAudioMetrics, - 'primaryMetricsBasis': 'UNIQUE_AUDIO_SHA256_DEDUPLICATED', - 'wrongConfidentHumanResults': wrongConfidentCases, - 'newlyIntroducedWrongConfidentHumanResults': - newlyIntroducedWrongConfident, - 'biasRecoveryReport': triggeredRecoveries, - 'permanentRegressions': [permanentRegression], - 'results': results, - 'recommendation': recommendation, - 'limitations': [ - 'Collection milestones are engineering targets, not statistical proof thresholds.', - 'The current four unique recordings and one speaker remain a labeled smoke test only.', - 'No general human/accent robustness or production-latency claim is supported.', - ], - }; - expect(newlyIntroducedWrongConfident, isEmpty); - - const jsonPath = - 'test/eval/entity_search/human_voice_corpus_benchmark_report.json'; - const markdownPath = - 'test/eval/entity_search/HUMAN_VOICE_CORPUS_BENCHMARK_REPORT.md'; - await File(jsonPath) - .writeAsString(const JsonEncoder.withIndent(' ').convert(report)); - await File(markdownPath).writeAsString(_markdown(report)); - print( - const JsonEncoder.withIndent(' ').convert({ - 'validation': { - 'valid': validation['valid'], - 'errors': validation['errors'], - 'warnings': validation['warnings'], - }, - 'uniqueAudioMetrics': uniqueAudioMetrics, - 'recommendation': recommendation, - 'reports': [jsonPath, markdownPath], - }), - ); - } finally { - speech.dispose(); - await db.close(); - } - }, timeout: const Timeout(Duration(minutes: 30))); -} - -Map _metrics(List> results) { - final scorable = results - .where((item) => item['canonicalScorable'] == true) - .toList(growable: false); - final rawEvaluations = results - .map((item) => item['pass1'] as Map) - .toList(growable: false); - final dynamicEvaluations = results - .map((item) => item['final'] as Map) - .toList(growable: false); - final rawScorable = scorable - .map((item) => item['pass1'] as Map) - .toList(growable: false); - final dynamicScorable = scorable - .map((item) => item['final'] as Map) - .toList(growable: false); - final deltas = { - 'IMPROVED': 0, - 'UNCHANGED': 0, - 'WORSENED': 0, - 'UNSCORABLE_AMBIGUOUS': 0, - }; - for (final result in results) { - final delta = (result['comparisonToRaw'] as Map)['delta'] as String; - deltas[delta] = (deltas[delta] ?? 0) + 1; - } - final triggerCount = results - .where((item) => item['secondPassTriggered'] == true) - .length; - final rawTotalLatency = results - .map((item) => ((item['pass1'] as Map)['latencyMs'] as Map)['total']) - .whereType() - .map((value) => value.toDouble()) - .toList(growable: false); - final rawSttLatency = results - .map((item) => ((item['pass1'] as Map)['latencyMs'] as Map)['stt']) - .whereType() - .map((value) => value.toDouble()) - .toList(growable: false); - final dynamicTotalLatency = results - .map((item) => (item['latencyMs'] as Map)['dynamicTotal']) - .whereType() - .map((value) => value.toDouble()) - .toList(growable: false); - - return { - 'files': results.length, - 'canonicalScorable': scorable.length, - 'RAW_BASELINE': _strategyMetrics(rawEvaluations, rawScorable), - 'RAW_DYNAMIC_TOP3': _strategyMetrics(dynamicEvaluations, dynamicScorable), - 'recoveryDelta': deltas, - 'secondPassTriggers': triggerCount, - 'secondPassTriggerRate': triggerCount / results.length, - 'averageSttCallsPerQuery': - results - .map((item) => item['sttCalls'] as int) - .reduce((left, right) => left + right) / - results.length, - 'latencyMs': { - 'RAW_BASELINE': _latencyStats(rawTotalLatency), - 'RAW_BASELINE_STT': _latencyStats(rawSttLatency), - 'RAW_DYNAMIC_TOP3': _latencyStats(dynamicTotalLatency), - }, - }; -} - -Map _strategyMetrics( - List evaluations, - List scorable, -) { - final wers = evaluations - .map((item) => item['wer']) - .whereType() - .map((value) => value.toDouble()) - .toList(growable: false); - final cers = evaluations - .map((item) => item['cer']) - .whereType() - .map((value) => value.toDouble()) - .toList(growable: false); - final outcomes = { - 'CORRECT_CONFIDENT': 0, - 'CORRECT_CLARIFICATION': 0, - 'WRONG_CONFIDENT': 0, - 'WRONG_CLARIFICATION': 0, - 'NO_MATCH': 0, - }; - for (final item in evaluations) { - final outcome = item['finalOutcome'] as String; - outcomes[outcome] = (outcomes[outcome] ?? 0) + 1; - } - return { - 'canonicalCorrect': scorable - .where((item) => item['canonicalOutcomeCorrect'] == true) - .length, - 'canonicalScorable': scorable.length, - 'canonicalCorrectRate': scorable.isEmpty - ? null - : scorable - .where((item) => item['canonicalOutcomeCorrect'] == true) - .length / - scorable.length, - 'outcomes': outcomes, - 'meanWer': _mean(wers), - 'meanCer': _mean(cers), - }; -} - -List> _wrongConfidentCases( - List> results, -) { - final cases = >[]; - for (final result in results) { - final raw = result['pass1'] as Map; - final dynamic = result['final'] as Map; - if (raw['wrongConfident'] == true) { - cases.add(_wrongCase(result, raw, 'RAW_BASELINE', false)); - } - if (dynamic['wrongConfident'] == true) { - cases.add( - _wrongCase( - result, - dynamic, - 'RAW_DYNAMIC_TOP3', - raw['wrongConfident'] != true, - ), - ); - } - } - return cases; -} - -Map _wrongCase( - Map result, - Map evaluation, - String strategy, - bool newlyIntroduced, -) => { - 'fixtureId': result['recordingId'], - 'filePath': result['audioFile'], - 'strategy': strategy, - 'transcript': (evaluation['transcription'] as Map)['transcript'], - 'expectedCanonicalId': result['expectedCanonicalEntityId'], - 'expectedCanonicalName': result['expectedCanonicalEntityName'], - 'resolvedCanonicalIds': - (evaluation['resolver'] as Map)['finalCanonicalEntityIds'], - 'resolvedCanonicalNames': - (evaluation['resolver'] as Map)['finalCanonicalEntityNames'], - 'newlyIntroducedByDynamic': newlyIntroduced, -}; - -Map _triggerSummary(Map result) { - final pass1 = result['pass1'] as Map; - final pass2 = result['pass2'] as Map; - final finalEvaluation = result['final'] as Map; - final circularEvidence = finalEvaluation['dynamicCircularEvidence']; - return { - 'fixtureId': result['recordingId'], - 'filePath': result['audioFile'], - 'pass1Transcript': (pass1['transcription'] as Map)['transcript'], - 'triggerReason': (result['triggerPolicy'] as Map)['reason'], - 'top3Hints': result['top3Hints'], - 'pass2Transcript': (pass2['transcription'] as Map)['transcript'], - 'rawOutcome': pass1['finalOutcome'], - 'dynamicOutcome': finalEvaluation['finalOutcome'], - 'delta': (result['comparisonToRaw'] as Map)['delta'], - 'circularEvidenceConfirmationRequired': - circularEvidence is Map && circularEvidence['guardApplied'] == true, - if (circularEvidence is Map) 'circularEvidence': circularEvidence, - }; -} - -String _recommendation(Map uniqueMetrics) { - final raw = uniqueMetrics['RAW_BASELINE'] as Map; - final dynamic = uniqueMetrics['RAW_DYNAMIC_TOP3'] as Map; - final delta = uniqueMetrics['recoveryDelta'] as Map; - final dynamicWrong = (dynamic['outcomes'] as Map)['WRONG_CONFIDENT'] as int; - final rawWrong = (raw['outcomes'] as Map)['WRONG_CONFIDENT'] as int; - if (dynamicWrong > rawWrong || (delta['WORSENED'] as int) > 0) { - return 'UNSAFE — DO NOT USE'; - } - if ((dynamic['canonicalCorrect'] as int) > (raw['canonicalCorrect'] as int) && - (delta['IMPROVED'] as int) > 0) { - return 'PROMISING — COLLECT MORE HUMAN AUDIO'; - } - return 'NO BENEFIT — KEEP BIASING SHELVED'; -} - -Map _latencyStats(List values) => { - 'average': _mean(values), - 'p50': _percentile(values, 0.50), - 'p95': _percentile(values, 0.95), -}; - -double? _mean(List values) => values.isEmpty - ? null - : values.reduce((left, right) => left + right) / values.length; - -double? _percentile(List values, double percentile) { - if (values.isEmpty) return null; - final sorted = [...values]..sort(); - final index = max(0, (percentile * sorted.length).ceil() - 1); - return sorted[index]; -} - -String _markdown(Map report) { - final validation = report['validation'] as Map; - final inventory = validation['audioInventory'] as Map; - final uniqueMetrics = report['uniqueAudioMetrics'] as Map; - final perFileMetrics = report['perFileMetrics'] as Map; - final raw = uniqueMetrics['RAW_BASELINE'] as Map; - final dynamic = uniqueMetrics['RAW_DYNAMIC_TOP3'] as Map; - final recovery = uniqueMetrics['recoveryDelta'] as Map; - final latency = uniqueMetrics['latencyMs'] as Map; - final rawLatency = latency['RAW_BASELINE'] as Map; - final dynamicLatency = latency['RAW_DYNAMIC_TOP3'] as Map; - final coverage = (report['corpusCoverage'] as Map)['uniqueAudio'] as Map; - final milestones = (report['collectionMilestones'] as List).whereType(); - final wrong = report['wrongConfidentHumanResults'] as List; - final newlyIntroducedWrong = - report['newlyIntroducedWrongConfidentHumanResults'] as List; - final triggers = (report['biasRecoveryReport'] as List).whereType(); - final buffer = StringBuffer() - ..writeln('# Human Voice Corpus Benchmark') - ..writeln() - ..writeln('`HUMAN_BENCHMARK_STATUS = LABELED_SMOKE_TEST_ONLY`') - ..writeln() - ..writeln('Architecture status: **FROZEN FOR HUMAN DATA COLLECTION**.') - ..writeln() - ..writeln('Recommendation: **${report['recommendation']}**') - ..writeln() - ..writeln('## Fixture validation') - ..writeln() - ..writeln('| Measure | Value |') - ..writeln('|---|---:|') - ..writeln('| Valid | ${validation['valid']} |') - ..writeln('| Errors | ${validation['errors']} |') - ..writeln('| Warnings | ${validation['warnings']} |') - ..writeln( - '| Fixtures silently dropped | ${validation['fixturesSilentlyDropped']} |', - ) - ..writeln('| TOTAL_FILES | ${inventory['TOTAL_FILES']} |') - ..writeln('| UNIQUE_AUDIO_FILES | ${inventory['UNIQUE_AUDIO_FILES']} |') - ..writeln( - '| DUPLICATE_GROUPS | ${(inventory['DUPLICATE_GROUPS'] as List).length} |', - ) - ..writeln() - ..writeln('## Primary unique-audio metrics') - ..writeln() - ..writeln('| Metric | RAW_BASELINE | RAW_DYNAMIC_TOP3 |') - ..writeln('|---|---:|---:|') - ..writeln( - '| Canonical correct | ${raw['canonicalCorrect']}/${raw['canonicalScorable']} | ${dynamic['canonicalCorrect']}/${dynamic['canonicalScorable']} |', - ) - ..writeln( - '| Correct confident | ${(raw['outcomes'] as Map)['CORRECT_CONFIDENT']} | ${(dynamic['outcomes'] as Map)['CORRECT_CONFIDENT']} |', - ) - ..writeln( - '| Correct clarification | ${(raw['outcomes'] as Map)['CORRECT_CLARIFICATION']} | ${(dynamic['outcomes'] as Map)['CORRECT_CLARIFICATION']} |', - ) - ..writeln( - '| WRONG_CONFIDENT | ${(raw['outcomes'] as Map)['WRONG_CONFIDENT']} | ${(dynamic['outcomes'] as Map)['WRONG_CONFIDENT']} |', - ) - ..writeln( - '| Wrong clarification | ${(raw['outcomes'] as Map)['WRONG_CLARIFICATION']} | ${(dynamic['outcomes'] as Map)['WRONG_CLARIFICATION']} |', - ) - ..writeln( - '| No-match | ${(raw['outcomes'] as Map)['NO_MATCH']} | ${(dynamic['outcomes'] as Map)['NO_MATCH']} |', - ) - ..writeln( - '| Mean WER | ${_percent(raw['meanWer'])} | ${_percent(dynamic['meanWer'])} |', - ) - ..writeln( - '| Mean CER | ${_percent(raw['meanCer'])} | ${_percent(dynamic['meanCer'])} |', - ) - ..writeln() - ..writeln( - 'Recovery: **${recovery['IMPROVED']} improved, ${recovery['UNCHANGED']} unchanged, ${recovery['WORSENED']} worsened**.', - ) - ..writeln() - ..writeln( - 'Second-pass trigger rate: ${_percent(uniqueMetrics['secondPassTriggerRate'])}; average STT calls/query: ${(uniqueMetrics['averageSttCallsPerQuery'] as num).toStringAsFixed(3)}.', - ) - ..writeln() - ..writeln('| Total latency | RAW_BASELINE | RAW_DYNAMIC_TOP3 |') - ..writeln('|---|---:|---:|') - ..writeln( - '| Average | ${_milliseconds(rawLatency['average'])} | ${_milliseconds(dynamicLatency['average'])} |', - ) - ..writeln( - '| p50 | ${_milliseconds(rawLatency['p50'])} | ${_milliseconds(dynamicLatency['p50'])} |', - ) - ..writeln( - '| p95 | ${_milliseconds(rawLatency['p95'])} | ${_milliseconds(dynamicLatency['p95'])} |', - ) - ..writeln() - ..writeln( - 'Per-file metrics are retained in JSON (${perFileMetrics['files']} files); primary metrics are SHA-256-deduplicated.', - ) - ..writeln() - ..writeln('## WRONG_CONFIDENT safety') - ..writeln() - ..writeln( - wrong.isEmpty - ? '**No wrong-confident human results.**' - : '**${wrong.length} wrong-confident human results — inspect individually below.**', - ) - ..writeln(); - for (final item in wrong.whereType()) { - buffer.writeln( - '- `${item['fixtureId']}` / `${item['strategy']}`: ${item['transcript']} → ${item['resolvedCanonicalNames']}', - ); - } - buffer - ..writeln() - ..writeln('### Newly introduced by RAW_DYNAMIC_TOP3') - ..writeln() - ..writeln( - newlyIntroducedWrong.isEmpty - ? '**None.**' - : '**${newlyIntroducedWrong.length} newly introduced wrong-confident results — listed individually below.**', - ); - for (final item in newlyIntroducedWrong.whereType()) { - buffer.writeln( - '- `${item['fixtureId']}`: ${item['transcript']} → ${item['resolvedCanonicalNames']}', - ); - } - buffer - ..writeln() - ..writeln('## Bias recovery triggers') - ..writeln(); - if (triggers.isEmpty) buffer.writeln('No second-pass triggers.'); - for (final item in triggers) { - buffer - ..writeln('### ${item['fixtureId']}') - ..writeln() - ..writeln('- Pass 1: ${item['pass1Transcript']}') - ..writeln('- Trigger: `${item['triggerReason']}`') - ..writeln('- Top 3: ${(item['top3Hints'] as List).join('; ')}') - ..writeln('- Pass 2: ${item['pass2Transcript']}') - ..writeln( - '- Outcome: `${item['rawOutcome']} → ${item['dynamicOutcome']}` (`${item['delta']}`)', - ) - ..writeln( - '- `CIRCULAR_EVIDENCE_CONFIRMATION_REQUIRED = ${item['circularEvidenceConfirmationRequired']}`', - ) - ..writeln(); - } - buffer - ..writeln('## Corpus coverage — unique audio') - ..writeln() - ..writeln('```json') - ..writeln(const JsonEncoder.withIndent(' ').convert(coverage)) - ..writeln('```') - ..writeln() - ..writeln('## Collection milestones') - ..writeln() - ..writeln( - '| Milestone | Unique recordings | Speakers | Remaining | Reached |', - ) - ..writeln('|---|---:|---:|---:|---|'); - for (final milestone in milestones) { - buffer.writeln( - '| ${milestone['milestone']} | ${milestone['currentUniqueRecordings']}/${milestone['targetUniqueRecordings']} | ${milestone['currentSpeakers']}/${milestone['targetSpeakers']} | ${milestone['uniqueRecordingsRemaining']} recordings, ${milestone['speakersRemaining']} speakers | ${milestone['reached']} |', - ); - } - buffer - ..writeln() - ..writeln( - 'These are engineering collection milestones, not statistical proof thresholds.', - ) - ..writeln() - ..writeln('## Permanent regression') - ..writeln() - ..writeln( - '`asim1.wav` remains `ES8A_ASIM1_DYNAMIC_TOP3_RECOVERY`: frozen RAW “Alex\'s” → `NO_MATCH`; dynamic “Alūksne Rally” → guarded `CORRECT_CLARIFICATION`.', - ) - ..writeln() - ..writeln('## One-command workflow') - ..writeln() - ..writeln('```bash') - ..writeln(report['oneCommandWorkflow']) - ..writeln('```') - ..writeln() - ..writeln( - 'The command validates every fixture, runs RAW and dynamic top-3, and regenerates this Markdown file plus the machine-readable JSON report.', - ); - return buffer.toString(); -} - -String _percent(Object? value) => - value is num ? '${(value.toDouble() * 100).toStringAsFixed(2)}%' : 'n/a'; - -String _milliseconds(Object? value) => - value is num ? '${value.toDouble().toStringAsFixed(1)} ms' : 'n/a'; diff --git a/test/eval/entity_search/human_voice_dynamic_top3_benchmark_test.dart b/test/eval/entity_search/human_voice_dynamic_top3_benchmark_test.dart deleted file mode 100644 index 75f34c8..0000000 --- a/test/eval/entity_search/human_voice_dynamic_top3_benchmark_test.dart +++ /dev/null @@ -1,346 +0,0 @@ -// ignore_for_file: avoid_print -@Tags(['live-db', 'live-api', 'benchmark']) -library; - -import 'dart:convert'; -import 'dart:io'; - -import 'package:ai_rally_search/services/database_service.dart'; -import 'package:ai_rally_search/services/entity_search/controlled_fallback_entity_resolver.dart'; -import 'package:ai_rally_search/services/entity_search/entity_search_lookup_adapter.dart'; -import 'package:ai_rally_search/services/entity_search/in_memory_entity_search_service.dart'; -import 'package:ai_rally_search/services/entity_search/mysql_entity_search_data_source.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/database_entity_resolver.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/entity_lookup_repository.dart'; -import 'package:ai_rally_search/services/llm/llm_query_parser_factory.dart'; -import 'package:ai_rally_search/services/speech/audio_preprocessor.dart'; -import 'package:ai_rally_search/services/speech/openai_speech_to_text_service.dart'; -import 'package:ai_rally_search/services/speech/speech_config.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:flutter_test/flutter_test.dart'; - -import 'human_voice_dynamic_top3_evaluator.dart'; -import 'human_voice_smoke_evaluator.dart'; - -void main() { - test('ES-8A real-human RAW plus dynamic top-3 second pass', () async { - await dotenv.load(fileName: '.env'); - final apiKey = dotenv.env['OPENAI_API_KEY']; - expect(apiKey, isNotNull); - expect(apiKey, isNotEmpty); - - final manifest = jsonDecode( - await File('test/eval/entity_search/human_voice_smoke_manifest.json') - .readAsString(), - ) as Map; - final fixtures = (manifest['fixtures'] as List) - .cast>(); - expect(fixtures, hasLength(5)); - expect(manifest['uniqueWaveformCount'], 4); - - final baselineReportFile = File( - 'test/eval/entity_search/human_voice_smoke_baseline_report.json', - ); - final baselineReport = jsonDecode( - await baselineReportFile.readAsString(), - ) as Map; - final frozenResults = (baselineReport['results'] as List) - .cast>(); - expect(frozenResults, hasLength(5)); - expect( - (baselineReport['uniqueAudioMetrics'] - as Map)['canonicalOutcomeCorrectIncludingClarification'], - 2, - ); - expect( - (baselineReport['uniqueAudioMetrics'] as Map)['canonicalOutcomeScorable'], - 3, - ); - - final diagnosisFile = File( - 'test/eval/entity_search/es8a_raw_failure_diagnosis.json', - ); - final diagnosis = - jsonDecode(await diagnosisFile.readAsString()) as Map; - expect(diagnosis['recordedBeforeDynamicBiasingImplementation'], isTrue); - expect( - (diagnosis['failedRecording'] as Map)['recordingId'], - 'human-smoke-001', - ); - - const audioPreprocessor = NoOpAudioPreprocessor(); - for (final fixture in fixtures) { - final audioFile = File(fixture['audioFile'] as String); - final original = await audioFile.readAsBytes(); - final processed = await audioPreprocessor.process( - inputBytes: original, - filename: audioFile.uri.pathSegments.last, - strategy: AudioPreprocessingStrategy.raw, - ); - expect(processed.changed, isFalse); - expect(processed.bytes, orderedEquals(original)); - expect(await audioFile.readAsBytes(), orderedEquals(original)); - } - - final db = DatabaseService(); - final speech = OpenAiSpeechToTextService( - config: SpeechConfig( - providerType: SpeechProviderType.openAiDirectDev, - endpointUrl: 'https://api.openai.com/v1/audio/transcriptions', - apiKey: apiKey, - model: 'gpt-transcribe', - timeout: const Duration(seconds: 45), - ), - ); - try { - final entities = await MySqlEntitySearchDataSource(database: db) - .loadEntities(); - final indexedIds = entities.map((item) => item.canonicalId).toSet(); - expect(indexedIds, contains('0cea6942-72e3-4257-a8c1-0f8148747d82')); - expect( - indexedIds, - contains('person:account:cf3ddf9c-a64b-4f59-a5e4-5230c44b4d87'), - ); - final entitySearch = InMemoryEntitySearchService.fromEntities(entities); - final legacy = DatabaseEntityLookupRepository(dbService: db); - final resolver = ControlledFallbackEntityResolver( - legacyResolver: DatabaseEntityResolver(repository: legacy), - entitySearchResolver: DatabaseEntityResolver( - repository: EntitySearchLookupAdapter( - searchService: entitySearch, - cityFallback: legacy, - ), - ), - config: const EntitySearchFallbackConfig( - mode: EntitySearchFallbackMode.fallback, - ), - ); - final pipeline = HumanVoiceSmokeEvaluator( - speech: speech, - parser: LlmQueryParserFactory.create(), - resolver: resolver, - entitySearch: entitySearch, - ); - final evaluator = HumanVoiceDynamicTop3Evaluator(pipeline: pipeline); - final results = >[]; - for (final fixture in fixtures) { - final frozen = frozenResults.firstWhere( - (item) => item['recordingId'] == fixture['recordingId'], - ); - results.add( - await evaluator.evaluate(fixture: fixture, frozenRawBaseline: frozen), - ); - } - - final uniqueResults = results - .where((item) => item['recordingId'] != 'human-smoke-003') - .toList(growable: false); - final perFileMetrics = _metrics(results); - final uniqueAudioMetrics = _metrics(uniqueResults); - final recommendation = _recommendation(uniqueAudioMetrics); - final report = { - 'phase': 'ES-8A', - 'experiment': 'REAL_HUMAN_DYNAMIC_TOP3_BIASING_SMOKE_TEST', - 'humanBenchmarkStatus': 'LABELED_SMOKE_TEST_ONLY', - 'realHumanAudio': true, - 'productionVoiceRoutingChanged': false, - 'sttBiasingProductionEnabled': false, - 'audioPreprocessing': { - 'implementation': 'NoOpAudioPreprocessor', - 'enabled': false, - 'originalAudioOverwritten': false, - }, - 'biasingConfiguration': { - 'strategy': 'DYNAMIC_TOP3_SECOND_PASS_ONLY', - 'topK': 3, - 'staticDomainContextUsed': false, - 'maximumSecondPasses': 1, - 'sameOriginalAudioRetranscribed': true, - 'circularExactEvidenceMayAutoConfirm': false, - }, - 'audioInventory': { - 'TOTAL_FILES': 5, - 'UNIQUE_AUDIO_FILES': 4, - 'DUPLICATE_GROUPS': [ - { - 'representative': 'human-smoke-001', - 'members': ['human-smoke-001', 'human-smoke-003'], - 'byteIdenticalVerified': true, - }, - ], - }, - 'canonicalGroundTruth': { - 'rallyAluksneEventId': '0cea6942-72e3-4257-a8c1-0f8148747d82', - 'maxFreemanCanonicalPersonId': - 'person:account:cf3ddf9c-a64b-4f59-a5e4-5230c44b4d87', - 'ambiguousCityFixtureForcedToEvent': false, - }, - 'frozenRawBaseline': { - 'report': baselineReportFile.path, - 'sha256': diagnosis['frozenRawBaselineSha256'], - 'diagnosis': diagnosisFile.path, - }, - 'perFileMetrics': perFileMetrics, - 'uniqueAudioMetrics': uniqueAudioMetrics, - 'results': results, - 'existingSyntheticEvidence': { - 'report': 'test/eval/entity_search/synthetic_stt_biasing_report.json', - 'rerun': false, - 'dynamicTop3WasBestSafetyUtilityConfiguration': true, - 'remainedExperimental': true, - }, - 'safety': { - 'requiredDynamicWrongConfident': 0, - 'actualPerFileDynamicWrongConfident': - perFileMetrics['dynamicWrongConfident'], - 'actualUniqueDynamicWrongConfident': - uniqueAudioMetrics['dynamicWrongConfident'], - 'blocker': (uniqueAudioMetrics['dynamicWrongConfident'] as int) > 0, - 'previouslyCorrectBaselineCasesRetranscribed': results.where((item) { - final comparison = item['comparisonToFrozenRaw'] as Map; - return comparison['baselineCanonicalOutcomeCorrect'] == true && - item['secondPassTriggered'] == true; - }).length, - }, - 'recommendation': recommendation, - 'limitations': [ - 'Four unique human waveforms are a labeled smoke test only.', - 'No production latency or human/accent robustness claim can be extrapolated.', - 'Query Understanding is LLM-backed and can vary for identical transcripts.', - 'Dynamically hinted exact spellings are circular evidence and are downgraded to clarification when they would otherwise auto-resolve.', - ], - }; - - expect(perFileMetrics['dynamicWrongConfident'], 0); - expect(uniqueAudioMetrics['dynamicWrongConfident'], 0); - for (final recordingId in ['human-smoke-004', 'human-smoke-005']) { - final item = results.firstWhere( - (result) => result['recordingId'] == recordingId, - ); - expect( - item['secondPassTriggered'], - isFalse, - reason: 'The existing safe human winner must not be retranscribed.', - ); - } - expect( - results.every((item) => (item['top3Hints'] as List).length <= 3), - isTrue, - ); - expect(results.every((item) => (item['sttCalls'] as int) <= 2), isTrue); - - const outputPath = - 'test/eval/entity_search/human_voice_dynamic_top3_report.json'; - await File(outputPath) - .writeAsString(const JsonEncoder.withIndent(' ').convert(report)); - print(const JsonEncoder.withIndent(' ').convert(report)); - } finally { - speech.dispose(); - await db.close(); - } - }, timeout: const Timeout(Duration(minutes: 20))); -} - -Map _metrics(List> results) { - final scorable = results - .where((item) => item['canonicalScorable'] == true) - .toList(growable: false); - final triggered = results - .where((item) => item['secondPassTriggered'] == true) - .toList(growable: false); - final deltas = { - 'IMPROVED': 0, - 'UNCHANGED': 0, - 'WORSENED': 0, - 'UNSCORABLE_AMBIGUOUS': 0, - }; - final baselineOutcomes = {}; - final dynamicOutcomes = {}; - for (final item in results) { - final comparison = item['comparisonToFrozenRaw'] as Map; - final delta = comparison['delta'] as String; - deltas[delta] = (deltas[delta] ?? 0) + 1; - final baseline = comparison['baselineOutcome'] as String; - final dynamic = comparison['dynamicOutcome'] as String; - baselineOutcomes[baseline] = (baselineOutcomes[baseline] ?? 0) + 1; - dynamicOutcomes[dynamic] = (dynamicOutcomes[dynamic] ?? 0) + 1; - } - final baselineStt = results - .map((item) => (item['latencyMs'] as Map)['frozenBaselineStt']) - .whereType(); - final baselineTotal = results - .map((item) => (item['latencyMs'] as Map)['frozenBaselineTotal']) - .whereType(); - final pass1Stt = results - .map((item) => (item['latencyMs'] as Map)['dynamicPass1Stt']) - .whereType(); - final pass1Total = results - .map((item) => (item['latencyMs'] as Map)['dynamicPass1Total']) - .whereType(); - final secondPassStt = triggered - .map((item) => (item['latencyMs'] as Map)['dynamicSecondPassStt']) - .whereType(); - final dynamicTotal = results - .map((item) => (item['latencyMs'] as Map)['dynamicTotal']) - .whereType(); - - return { - 'files': results.length, - 'canonicalScorable': scorable.length, - 'baselineCanonicalCorrect': scorable.where((item) { - final comparison = item['comparisonToFrozenRaw'] as Map; - return comparison['baselineCanonicalOutcomeCorrect'] == true; - }).length, - 'dynamicCanonicalCorrect': scorable.where((item) { - final comparison = item['comparisonToFrozenRaw'] as Map; - return comparison['dynamicCanonicalOutcomeCorrect'] == true; - }).length, - 'dynamicWrongConfident': scorable.where((item) { - final comparison = item['comparisonToFrozenRaw'] as Map; - return comparison['dynamicWrongConfident'] == true; - }).length, - 'secondPassTriggers': triggered.length, - 'secondPassTriggerRate': triggered.length / results.length, - 'averageSttCallsPerQuery': - results - .map((item) => item['sttCalls'] as int) - .reduce((left, right) => left + right) / - results.length, - 'recoveryDelta': deltas, - 'baselineOutcomes': baselineOutcomes, - 'dynamicOutcomes': dynamicOutcomes, - 'circularEvidenceGuardsApplied': results.where((item) { - final finalEvaluation = item['final'] as Map; - final evidence = finalEvaluation['dynamicCircularEvidence']; - return evidence is Map && evidence['guardApplied'] == true; - }).length, - 'latencyMs': { - 'averageFrozenBaselineStt': _mean(baselineStt), - 'averageFrozenBaselineTotal': _mean(baselineTotal), - 'averageDynamicPass1Stt': _mean(pass1Stt), - 'averageDynamicPass1Total': _mean(pass1Total), - 'averageDynamicSecondPassSttWhenTriggered': _mean(secondPassStt), - 'averageDynamicTotal': _mean(dynamicTotal), - }, - }; -} - -String _recommendation(Map uniqueMetrics) { - if ((uniqueMetrics['dynamicWrongConfident'] as int) > 0 || - ((uniqueMetrics['recoveryDelta'] as Map)['WORSENED'] as int) > 0) { - return 'UNSAFE — DO NOT USE'; - } - final baseline = uniqueMetrics['baselineCanonicalCorrect'] as int; - final dynamic = uniqueMetrics['dynamicCanonicalCorrect'] as int; - final improved = (uniqueMetrics['recoveryDelta'] as Map)['IMPROVED'] as int; - if (dynamic > baseline && improved > 0) { - return 'PROMISING — COLLECT MORE HUMAN AUDIO'; - } - return 'NO BENEFIT — KEEP BIASING SHELVED'; -} - -double? _mean(Iterable values) { - final list = values.map((value) => value.toDouble()).toList(growable: false); - if (list.isEmpty) return null; - return list.reduce((left, right) => left + right) / list.length; -} diff --git a/test/eval/entity_search/human_voice_preprocessing_ab_test.dart b/test/eval/entity_search/human_voice_preprocessing_ab_test.dart deleted file mode 100644 index 621479f..0000000 --- a/test/eval/entity_search/human_voice_preprocessing_ab_test.dart +++ /dev/null @@ -1,540 +0,0 @@ -// ignore_for_file: avoid_print -@Tags(['live-db', 'live-api', 'benchmark']) -library; - -import 'dart:convert'; -import 'dart:io'; -import 'dart:math'; - -import 'package:ai_rally_search/services/database_service.dart'; -import 'package:ai_rally_search/services/entity_search/controlled_fallback_entity_resolver.dart'; -import 'package:ai_rally_search/services/entity_search/entity_search_lookup_adapter.dart'; -import 'package:ai_rally_search/services/entity_search/in_memory_entity_search_service.dart'; -import 'package:ai_rally_search/services/entity_search/mysql_entity_search_data_source.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/database_entity_resolver.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/entity_lookup_repository.dart'; -import 'package:ai_rally_search/services/llm/llm_query_parser_factory.dart'; -import 'package:ai_rally_search/services/speech/audio_preprocessor.dart'; -import 'package:ai_rally_search/services/speech/openai_speech_to_text_service.dart'; -import 'package:ai_rally_search/services/speech/speech_config.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:flutter_test/flutter_test.dart'; - -import 'human_voice_smoke_evaluator.dart'; -import 'pcm16_wav.dart'; - -void main() { - test('ES-7 conservative preprocessing A/B on five human fixtures', () async { - await dotenv.load(fileName: '.env'); - final apiKey = dotenv.env['OPENAI_API_KEY']; - expect(apiKey, isNotNull); - expect(apiKey, isNotEmpty); - - final baselineFile = File( - 'test/eval/entity_search/human_voice_smoke_baseline_report.json', - ); - expect( - baselineFile.existsSync(), - isTrue, - reason: 'The RAW baseline must be frozen before preprocessing A/B.', - ); - final baseline = - jsonDecode(await baselineFile.readAsString()) as Map; - expect(baseline['rawBaselineCapturedBeforePreprocessing'], isTrue); - final rawResults = (baseline['results'] as List) - .cast>(); - expect(rawResults, hasLength(5)); - - final manifest = jsonDecode( - await File('test/eval/entity_search/human_voice_smoke_manifest.json') - .readAsString(), - ) as Map; - final fixtures = (manifest['fixtures'] as List) - .cast>(); - - final db = DatabaseService(); - final speech = OpenAiSpeechToTextService( - config: SpeechConfig( - providerType: SpeechProviderType.openAiDirectDev, - endpointUrl: 'https://api.openai.com/v1/audio/transcriptions', - apiKey: apiKey, - model: 'gpt-transcribe', - timeout: const Duration(seconds: 45), - ), - ); - try { - final entities = await MySqlEntitySearchDataSource(database: db) - .loadEntities(); - final entitySearch = InMemoryEntitySearchService.fromEntities(entities); - final legacy = DatabaseEntityLookupRepository(dbService: db); - final resolver = ControlledFallbackEntityResolver( - legacyResolver: DatabaseEntityResolver(repository: legacy), - entitySearchResolver: DatabaseEntityResolver( - repository: EntitySearchLookupAdapter( - searchService: entitySearch, - cityFallback: legacy, - ), - ), - config: const EntitySearchFallbackConfig( - mode: EntitySearchFallbackMode.fallback, - ), - ); - final evaluator = HumanVoiceSmokeEvaluator( - speech: speech, - parser: LlmQueryParserFactory.create(), - resolver: resolver, - entitySearch: entitySearch, - ); - const preprocessor = SpeechAudioPreprocessor(); - final processedResults = >[]; - final comparisons = >[]; - - for (final fixture in fixtures) { - final recordingId = fixture['recordingId'] as String; - final original = File(fixture['audioFile'] as String); - final originalBytes = await original.readAsBytes(); - final raw = rawResults.firstWhere( - (item) => item['recordingId'] == recordingId, - ); - for (final strategy in AudioPreprocessingStrategy.values.where( - (strategy) => strategy != AudioPreprocessingStrategy.raw, - )) { - final processed = await preprocessor.process( - inputBytes: originalBytes, - filename: original.uri.pathSegments.last, - strategy: strategy, - ); - final outputWave = Pcm16Wav.decode(processed.bytes); - final result = await evaluator.evaluateProcessed( - recordingId: recordingId, - originalAudioFile: original, - processed: processed, - groundTruth: fixture, - ); - result['derivedAudio'] = { - 'storage': 'memory_only_not_persisted', - 'originalOverwritten': false, - ...outputWave.diagnostics(fileSizeBytes: processed.bytes.length), - }; - processedResults.add(result); - comparisons.add(_compare(raw, result)); - } - expect(await original.readAsBytes(), orderedEquals(originalBytes)); - } - - final uniqueComparisons = comparisons - .where((item) => item['recordingId'] != 'human-smoke-003') - .toList(growable: false); - final allRuns = [...rawResults, ...processedResults]; - final uniqueRuns = allRuns - .where((item) => item['recordingId'] != 'human-smoke-003') - .toList(growable: false); - final report = { - 'phase': 'ES-7', - 'realHumanAudio': true, - 'humanSampleCount': fixtures.length, - 'uniqueWaveformCount': manifest['uniqueWaveformCount'], - 'humanBenchmarkStatus': 'LABELED_SMOKE_TEST_ONLY', - 'sttBiasingUsed': false, - 'productionVoiceBehaviorChanged': false, - 'originalAudioOverwritten': false, - 'rawBaselineReport': baselineFile.path, - 'audioInventory': { - 'TOTAL_FILES': fixtures.length, - 'UNIQUE_AUDIO_FILES': manifest['uniqueWaveformCount'], - 'DUPLICATE_GROUPS': [ - { - 'representative': 'human-smoke-001', - 'members': ['human-smoke-001', 'human-smoke-003'], - 'byteIdenticalVerified': true, - }, - ], - }, - 'rawResults': rawResults, - 'preprocessingStrategies': [ - 'VAD_ONLY: energy-based silence trim with 150 ms padding', - 'NORMALIZED: bounded RMS normalization, -23 dBFS target, -3 dBFS peak ceiling, +9.54 dB maximum gain', - 'NOISE_SUPPRESSED: conservative 10 ms soft noise gate, approximately -12 dB below adaptive threshold', - 'VAD_NORMALIZED_NOISE_SUPPRESSED: noise gate then normalization then silence trim', - ], - 'processedResults': processedResults, - 'comparisonsToFrozenRaw': comparisons, - 'perFileStrategyMetrics': _strategyMetrics(comparisons), - 'uniqueAudioStrategyMetrics': _strategyMetrics(uniqueComparisons), - 'scoringAvailability': { - 'canonicalCorrectness': true, - 'canonicalFixtures': fixtures - .where((item) => item['canonicalEntityId'] != null) - .length, - 'targetCandidateRank': true, - 'correctConfident': true, - 'wrongConfident': true, - 'werCer': true, - 'transcriptFixtures': fixtures.length, - 'limitation': 'human-smoke-002 is a city query without a canonical city ID, so its canonical outcome remains unscored.', - }, - 'voiceExactMatchEscalation': { - 'status': 'ASSESSED_WHERE_CANONICAL_GROUND_TRUTH_EXISTS', - 'perFileAssessableRuns': allRuns - .where((item) => item['expectedCanonicalEntityId'] != null) - .length, - 'uniqueAudioAssessableRuns': uniqueRuns - .where((item) => item['expectedCanonicalEntityId'] != null) - .length, - 'perFileConfirmedCases': allRuns.where((item) { - final finding = item['voiceExactMatchEscalation']; - return finding is Map && finding['occurred'] == true; - }).length, - 'uniqueAudioConfirmedCases': uniqueRuns.where((item) { - final finding = item['voiceExactMatchEscalation']; - return finding is Map && finding['occurred'] == true; - }).length, - }, - 'optionalProviderComparison': { - 'performed': false, - 'reason': 'No second non-mock STT provider is already supported by the project.', - }, - 'recommendation': _recommendation(uniqueComparisons), - 'limitations': [ - 'Five files are still a smoke test, not a statistically meaningful benchmark.', - 'Only four unique waveforms exist because human-smoke-003 is byte-identical to human-smoke-001.', - 'The city fixture has transcript ground truth but no canonical city ID.', - 'The current LLM query parser can vary for identical transcripts, which is a downstream confound visible in the frozen RAW duplicate pair.', - ], - }; - const outputPath = - 'test/eval/entity_search/human_voice_preprocessing_ab_report.json'; - await File(outputPath) - .writeAsString(const JsonEncoder.withIndent(' ').convert(report)); - print(const JsonEncoder.withIndent(' ').convert(report)); - } finally { - speech.dispose(); - await db.close(); - } - }, timeout: const Timeout(Duration(minutes: 20))); -} - -Map _compare( - Map raw, - Map processed, -) { - final rawTranscript = - ((raw['transcription'] as Map)['transcript'] as String?) ?? ''; - final processedTranscript = - ((processed['transcription'] as Map)['transcript'] as String?) ?? ''; - final rawResolver = raw['resolver'] as Map; - final processedResolver = processed['resolver'] as Map; - final rawMentions = _mentions(raw); - final processedMentions = _mentions(processed); - final unchanged = - rawTranscript == processedTranscript && - rawResolver['decision'] == processedResolver['decision'] && - _sameStrings( - (rawResolver['finalCanonicalEntityIds'] as List).cast(), - (processedResolver['finalCanonicalEntityIds'] as List).cast(), - ); - final rawCanonicalCorrect = raw['canonicalEntityCorrect'] as bool?; - final processedCanonicalCorrect = - processed['canonicalEntityCorrect'] as bool?; - final rawCanonicalOutcomeCorrect = raw['canonicalOutcomeCorrect'] as bool?; - final processedCanonicalOutcomeCorrect = - processed['canonicalOutcomeCorrect'] as bool?; - final rawWrongConfident = raw['wrongConfident'] as bool?; - final processedWrongConfident = processed['wrongConfident'] as bool?; - final rawWer = (raw['wer'] as num?)?.toDouble(); - final processedWer = (processed['wer'] as num?)?.toDouble(); - final rawCer = (raw['cer'] as num?)?.toDouble(); - final processedCer = (processed['cer'] as num?)?.toDouble(); - final assessment = _assessment( - unchanged: unchanged, - rawCanonicalCorrect: rawCanonicalOutcomeCorrect, - processedCanonicalCorrect: processedCanonicalOutcomeCorrect, - rawWrongConfident: rawWrongConfident, - processedWrongConfident: processedWrongConfident, - rawFinalOutcome: raw['finalOutcome'] as String?, - processedFinalOutcome: processed['finalOutcome'] as String?, - rawEntityMentionExact: raw['entityMentionExact'] as bool?, - processedEntityMentionExact: processed['entityMentionExact'] as bool?, - rawWer: rawWer, - processedWer: processedWer, - rawCer: rawCer, - processedCer: processedCer, - ); - return { - 'recordingId': processed['recordingId'], - 'strategy': processed['strategy'], - 'rawTranscript': rawTranscript, - 'processedTranscript': processedTranscript, - 'transcriptUnchanged': rawTranscript == processedTranscript, - 'tokenEditRateFromRaw': _editRate( - rawTranscript.split(RegExp(r'\s+')), - processedTranscript.split(RegExp(r'\s+')), - ), - 'characterEditRateFromRaw': _editRate( - rawTranscript.runes.toList(), - processedTranscript.runes.toList(), - ), - 'rawEntityMentions': rawMentions, - 'processedEntityMentions': processedMentions, - 'entityMentionChanged': !_sameStrings(rawMentions, processedMentions), - 'rawResolverDecision': rawResolver['decision'], - 'processedResolverDecision': processedResolver['decision'], - 'resolverOutcomeChanged': - rawResolver['decision'] != processedResolver['decision'], - 'rawFinalCanonicalEntityIds': rawResolver['finalCanonicalEntityIds'], - 'processedFinalCanonicalEntityIds': - processedResolver['finalCanonicalEntityIds'], - 'rawCanonicalEntityCorrect': rawCanonicalCorrect, - 'processedCanonicalEntityCorrect': processedCanonicalCorrect, - 'rawCanonicalOutcomeCorrect': rawCanonicalOutcomeCorrect, - 'processedCanonicalOutcomeCorrect': processedCanonicalOutcomeCorrect, - 'rawEntityMentionExact': raw['entityMentionExact'], - 'processedEntityMentionExact': processed['entityMentionExact'], - 'rawFinalOutcome': raw['finalOutcome'], - 'processedFinalOutcome': processed['finalOutcome'], - 'rawTargetCandidateRank': raw['targetCandidateRank'], - 'processedTargetCandidateRank': processed['targetCandidateRank'], - 'rawFinalSearchQuery': (raw['querySemantics'] as Map?)?['finalSearchQuery'], - 'processedFinalSearchQuery': - (processed['querySemantics'] as Map?)?['finalSearchQuery'], - 'rawWrongConfident': rawWrongConfident, - 'processedWrongConfident': processedWrongConfident, - 'rawWer': rawWer, - 'processedWer': processedWer, - 'werDelta': rawWer == null || processedWer == null - ? null - : processedWer - rawWer, - 'rawCer': rawCer, - 'processedCer': processedCer, - 'cerDelta': rawCer == null || processedCer == null - ? null - : processedCer - rawCer, - 'observablePipelineOutputUnchanged': unchanged, - 'preprocessingRegressionAssessment': assessment, - }; -} - -String _assessment({ - required bool unchanged, - required bool? rawCanonicalCorrect, - required bool? processedCanonicalCorrect, - required bool? rawWrongConfident, - required bool? processedWrongConfident, - required String? rawFinalOutcome, - required String? processedFinalOutcome, - required bool? rawEntityMentionExact, - required bool? processedEntityMentionExact, - required double? rawWer, - required double? processedWer, - required double? rawCer, - required double? processedCer, -}) { - if (rawCanonicalCorrect != null && processedCanonicalCorrect != null) { - if (!rawCanonicalCorrect && processedCanonicalCorrect) { - return 'CANONICAL_IMPROVEMENT'; - } - if (rawCanonicalCorrect && !processedCanonicalCorrect) { - return 'CANONICAL_REGRESSION'; - } - if (rawWrongConfident != true && processedWrongConfident == true) { - return 'WRONG_CONFIDENT_REGRESSION'; - } - } - if (rawFinalOutcome != 'WRONG_CLARIFICATION' && - processedFinalOutcome == 'WRONG_CLARIFICATION') { - return 'WRONG_CLARIFICATION_REGRESSION'; - } - if (rawEntityMentionExact != null && processedEntityMentionExact != null) { - if (!rawEntityMentionExact && processedEntityMentionExact) { - return 'ENTITY_MENTION_IMPROVEMENT'; - } - if (rawEntityMentionExact && !processedEntityMentionExact) { - return 'ENTITY_MENTION_REGRESSION'; - } - } - if (rawWer != null && processedWer != null) { - final delta = processedWer - rawWer; - if (delta < -0.000001) return 'TRANSCRIPT_IMPROVEMENT'; - if (delta > 0.000001) return 'TRANSCRIPT_REGRESSION'; - } - if (rawCer != null && processedCer != null) { - final delta = processedCer - rawCer; - if (delta < -0.000001) return 'TRANSCRIPT_IMPROVEMENT'; - if (delta > 0.000001) return 'TRANSCRIPT_REGRESSION'; - } - return unchanged - ? 'NO_OBSERVABLE_CHANGE' - : 'CANONICAL_MENTION_AND_TRANSCRIPT_METRICS_UNCHANGED'; -} - -List> _strategyMetrics( - List> comparisons, -) { - final strategies = - comparisons.map((item) => item['strategy'] as String).toSet().toList() - ..sort(); - return strategies - .map((strategy) { - final rows = comparisons - .where((item) => item['strategy'] == strategy) - .toList(growable: false); - final werValues = rows - .map((item) => item['processedWer']) - .whereType() - .toList(growable: false); - final cerValues = rows - .map((item) => item['processedCer']) - .whereType() - .toList(growable: false); - return { - 'strategy': strategy, - 'runs': rows.length, - 'canonicalCorrect': rows - .where((item) => item['processedCanonicalOutcomeCorrect'] == true) - .length, - 'canonicalScorable': rows - .where((item) => item['processedCanonicalOutcomeCorrect'] != null) - .length, - 'finalCanonicalResolvedCorrect': rows - .where((item) => item['processedCanonicalEntityCorrect'] == true) - .length, - 'entityMentionExact': rows - .where((item) => item['processedEntityMentionExact'] == true) - .length, - 'wrongConfident': rows - .where((item) => item['processedWrongConfident'] == true) - .length, - 'meanWer': werValues.isEmpty - ? null - : werValues.reduce((left, right) => left + right) / - werValues.length, - 'meanCer': cerValues.isEmpty - ? null - : cerValues.reduce((left, right) => left + right) / - cerValues.length, - 'canonicalImprovements': rows - .where( - (item) => - item['preprocessingRegressionAssessment'] == - 'CANONICAL_IMPROVEMENT', - ) - .length, - 'canonicalRegressions': rows - .where( - (item) => - item['preprocessingRegressionAssessment'] == - 'CANONICAL_REGRESSION', - ) - .length, - 'outcomeRegressions': rows - .where( - (item) => - item['preprocessingRegressionAssessment'] == - 'WRONG_CLARIFICATION_REGRESSION', - ) - .length, - 'entityMentionImprovements': rows - .where( - (item) => - item['preprocessingRegressionAssessment'] == - 'ENTITY_MENTION_IMPROVEMENT', - ) - .length, - 'entityMentionRegressions': rows - .where( - (item) => - item['preprocessingRegressionAssessment'] == - 'ENTITY_MENTION_REGRESSION', - ) - .length, - 'transcriptImprovements': rows - .where( - (item) => - item['preprocessingRegressionAssessment'] == - 'TRANSCRIPT_IMPROVEMENT', - ) - .length, - 'transcriptRegressions': rows - .where( - (item) => - item['preprocessingRegressionAssessment'] == - 'TRANSCRIPT_REGRESSION', - ) - .length, - }; - }) - .toList(growable: false); -} - -String _recommendation(List> comparisons) { - final assessments = comparisons - .map((item) => item['preprocessingRegressionAssessment']) - .whereType() - .toList(growable: false); - final improvements = assessments - .where( - (item) => - item == 'CANONICAL_IMPROVEMENT' || - item == 'ENTITY_MENTION_IMPROVEMENT' || - item == 'TRANSCRIPT_IMPROVEMENT', - ) - .length; - final regressions = assessments - .where( - (item) => - item == 'CANONICAL_REGRESSION' || - item == 'WRONG_CONFIDENT_REGRESSION' || - item == 'WRONG_CLARIFICATION_REGRESSION' || - item == 'ENTITY_MENTION_REGRESSION' || - item == 'TRANSCRIPT_REGRESSION', - ) - .length; - if (improvements > 0 && regressions == 0) { - return 'AUDIO PREPROCESSING PROMISING'; - } - if (regressions > 0 && improvements == 0) { - return 'AUDIO PREPROCESSING HARMFUL'; - } - if (improvements == 0 && regressions == 0) { - return 'AUDIO PREPROCESSING LOW VALUE'; - } - return 'INSUFFICIENT / MIXED'; -} - -List _mentions(Map result) { - final understanding = result['queryUnderstanding']; - if (understanding is! Map || understanding['rawEntityMentions'] is! List) { - return const []; - } - return (understanding['rawEntityMentions'] as List) - .whereType() - .map((item) => '${item['entityType']}:${item['mention']}') - .toList(growable: false); -} - -bool _sameStrings(List left, List right) { - if (left.length != right.length) return false; - for (var index = 0; index < left.length; index++) { - if (left[index] != right[index]) return false; - } - return true; -} - -double _editRate(List reference, List hypothesis) { - if (reference.isEmpty) return hypothesis.isEmpty ? 0 : 1; - var previous = List.generate(hypothesis.length + 1, (index) => index); - for (var row = 1; row <= reference.length; row++) { - final current = List.filled(hypothesis.length + 1, 0)..[0] = row; - for (var column = 1; column <= hypothesis.length; column++) { - final substitution = - previous[column - 1] + - (reference[row - 1] == hypothesis[column - 1] ? 0 : 1); - current[column] = min( - min(previous[column] + 1, current[column - 1] + 1), - substitution, - ); - } - previous = current; - } - return previous.last / reference.length; -} diff --git a/test/eval/entity_search/human_voice_smoke_baseline_test.dart b/test/eval/entity_search/human_voice_smoke_baseline_test.dart deleted file mode 100644 index 9ad4a9f..0000000 --- a/test/eval/entity_search/human_voice_smoke_baseline_test.dart +++ /dev/null @@ -1,303 +0,0 @@ -// ignore_for_file: avoid_print -@Tags(['live-db', 'live-api', 'benchmark']) -library; - -import 'dart:convert'; -import 'dart:io'; - -import 'package:ai_rally_search/services/database_service.dart'; -import 'package:ai_rally_search/services/entity_search/controlled_fallback_entity_resolver.dart'; -import 'package:ai_rally_search/services/entity_search/entity_search_lookup_adapter.dart'; -import 'package:ai_rally_search/services/entity_search/in_memory_entity_search_service.dart'; -import 'package:ai_rally_search/services/entity_search/mysql_entity_search_data_source.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/database_entity_resolver.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/entity_lookup_repository.dart'; -import 'package:ai_rally_search/services/llm/llm_query_parser_factory.dart'; -import 'package:ai_rally_search/services/speech/openai_speech_to_text_service.dart'; -import 'package:ai_rally_search/services/speech/speech_config.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:flutter_test/flutter_test.dart'; - -import 'human_voice_smoke_evaluator.dart'; -import 'pcm16_wav.dart'; - -void main() { - test('ES-7 records immutable five-fixture real-human RAW baseline', () async { - await dotenv.load(fileName: '.env'); - final apiKey = dotenv.env['OPENAI_API_KEY']; - expect(apiKey, isNotNull); - expect(apiKey, isNotEmpty); - - final manifestFile = File( - 'test/eval/entity_search/human_voice_smoke_manifest.json', - ); - final manifest = - jsonDecode(await manifestFile.readAsString()) as Map; - final fixtures = (manifest['fixtures'] as List) - .cast>(); - expect(fixtures, hasLength(manifest['humanSampleCount'])); - expect(fixtures, hasLength(5)); - expect( - fixtures.every((item) => item['referenceTranscriptRaw'] != null), - isTrue, - ); - - final firstBytes = await File(fixtures[0]['audioFile'] as String) - .readAsBytes(); - final thirdBytes = await File(fixtures[2]['audioFile'] as String) - .readAsBytes(); - expect(firstBytes, orderedEquals(thirdBytes)); - - final db = DatabaseService(); - final speech = OpenAiSpeechToTextService( - config: SpeechConfig( - providerType: SpeechProviderType.openAiDirectDev, - endpointUrl: 'https://api.openai.com/v1/audio/transcriptions', - apiKey: apiKey, - model: 'gpt-transcribe', - timeout: const Duration(seconds: 45), - ), - ); - try { - final rallyTruthRows = await db.query(''' - SELECT event_id, event_name - FROM rally_events - WHERE event_name = 'Rally Alūksne 2026' - '''); - expect(rallyTruthRows, hasLength(1)); - expect( - rallyTruthRows.single['event_id'].toString(), - '0cea6942-72e3-4257-a8c1-0f8148747d82', - ); - - final maxDriverProfiles = await db.query(''' - SELECT driver_id, account_id, full_name - FROM user_driver_profile - WHERE LOWER(full_name) = 'max freeman' - '''); - final maxCodriverProfiles = await db.query(''' - SELECT codriver_id, account_id, full_name - FROM user_codriver_profile - WHERE LOWER(full_name) = 'max freeman' - '''); - expect(maxDriverProfiles, isEmpty); - expect(maxCodriverProfiles, hasLength(1)); - expect( - maxCodriverProfiles.single['account_id'].toString(), - 'cf3ddf9c-a64b-4f59-a5e4-5230c44b4d87', - ); - expect( - maxCodriverProfiles.single['codriver_id'].toString(), - '7a633b52-950e-49ef-8cab-34cd43e99366', - ); - final maxCodriverParticipation = await db.query(''' - SELECT DISTINCT ev.event_id - FROM rally_entry_list entry - INNER JOIN rally_sub_events sub - ON entry.sub_event_id = sub.sub_event_id - INNER JOIN rally_events ev ON sub.event_id = ev.event_id - WHERE entry.user_co_driver_id = '7a633b52-950e-49ef-8cab-34cd43e99366' - '''); - expect(maxCodriverParticipation, isNotEmpty); - - final entities = await MySqlEntitySearchDataSource(database: db) - .loadEntities(); - final indexedIds = entities.map((entity) => entity.canonicalId).toSet(); - final expectedCanonicalIds = fixtures - .map((fixture) => fixture['canonicalEntityId']) - .whereType() - .toSet(); - expect(indexedIds, containsAll(expectedCanonicalIds)); - final entitySearch = InMemoryEntitySearchService.fromEntities(entities); - final legacy = DatabaseEntityLookupRepository(dbService: db); - final resolver = ControlledFallbackEntityResolver( - legacyResolver: DatabaseEntityResolver(repository: legacy), - entitySearchResolver: DatabaseEntityResolver( - repository: EntitySearchLookupAdapter( - searchService: entitySearch, - cityFallback: legacy, - ), - ), - config: const EntitySearchFallbackConfig( - mode: EntitySearchFallbackMode.fallback, - ), - ); - final evaluator = HumanVoiceSmokeEvaluator( - speech: speech, - parser: LlmQueryParserFactory.create(), - resolver: resolver, - entitySearch: entitySearch, - ); - - final results = >[]; - final fixtureMetadata = >[]; - for (final fixture in fixtures) { - final audioFile = File(fixture['audioFile'] as String); - expect(audioFile.existsSync(), isTrue); - final bytes = await audioFile.readAsBytes(); - final wav = Pcm16Wav.decode(bytes); - fixtureMetadata.add({ - ...fixture, - ...wav.diagnostics(fileSizeBytes: bytes.length), - 'originalImmutable': true, - 'byteIdenticalDuplicateVerified': - fixture['duplicateAudioOf'] == null || - fixture['recordingId'] == 'human-smoke-001' - ? false - : bytes.length == firstBytes.length && - List.generate( - bytes.length, - (index) => index, - ).every((index) => bytes[index] == firstBytes[index]), - }); - results.add( - await evaluator.evaluateRaw( - recordingId: fixture['recordingId'] as String, - audioFile: audioFile, - groundTruth: fixture, - ), - ); - } - - final report = { - 'phase': 'ES-7', - 'realHumanAudio': true, - 'humanSampleCount': fixtures.length, - 'uniqueWaveformCount': manifest['uniqueWaveformCount'], - 'humanBenchmarkStatus': 'LABELED_SMOKE_TEST_ONLY', - 'rawBaselineCapturedBeforePreprocessing': true, - 'sttBiasingUsed': false, - 'productionVoiceBehaviorChanged': false, - 'provider': { - 'stt': 'OpenAI gpt-transcribe', - 'requestedLanguage': 'en', - 'detectedLanguageAvailable': false, - 'confidenceOrLogprobAvailable': results.any( - (item) => - ((item['transcription'] as Map)['confidenceAvailable'] == true), - ), - }, - 'audioInventory': { - 'TOTAL_FILES': fixtures.length, - 'UNIQUE_AUDIO_FILES': manifest['uniqueWaveformCount'], - 'DUPLICATE_GROUPS': [ - { - 'representative': 'human-smoke-001', - 'members': ['human-smoke-001', 'human-smoke-003'], - 'byteIdenticalVerified': true, - }, - ], - }, - 'independentLiveDbGroundTruth': { - 'derivedFromEntitySearchResults': false, - 'rallyAluksne': { - 'queryMethod': 'exact rally_events.event_name lookup', - 'eventId': rallyTruthRows.single['event_id'].toString(), - 'eventName': rallyTruthRows.single['event_name'].toString(), - 'interpretationAmbiguous': false, - }, - 'maxFreeman': { - 'queryMethod': 'exact driver/co-driver profile lookup plus raw entry-list participation joins', - 'canonicalPersonId': - 'person:account:${maxCodriverProfiles.single['account_id']}', - 'accountId': maxCodriverProfiles.single['account_id'].toString(), - 'driverProfileCount': maxDriverProfiles.length, - 'codriverProfileCount': maxCodriverProfiles.length, - 'codriverId': maxCodriverProfiles.single['codriver_id'].toString(), - 'driverParticipationCount': 0, - 'codriverParticipationCount': maxCodriverParticipation.length, - 'expectedQueryRoleSemantics': 'ANY', - 'actualLiveParticipationRole': 'CO_DRIVER_ONLY', - 'interpretationAmbiguous': false, - }, - }, - 'fixtureMetadata': fixtureMetadata, - 'groundTruth': { - 'status': 'HUMAN_SUPPLIED_FOR_ALL_5_TRANSCRIPTS', - 'source': manifest['groundTruthSource'], - 'referenceTranscriptsAvailable': fixtures - .where((item) => item['referenceTranscriptRaw'] != null) - .length, - 'entityMentionsAvailable': fixtures - .where((item) => item['entityMention'] != null) - .length, - 'canonicalEntityIdsAvailable': fixtures - .where((item) => item['canonicalEntityId'] != null) - .length, - 'entityTypesAvailable': fixtures - .where((item) => item['entityType'] != null) - .length, - 'labelsInferredFromStt': false, - 'labelsInferredFromWinningCandidate': false, - 'canonicalIdsVerifiedInLoadedIndex': true, - }, - 'results': results, - 'perFileMetrics': _metrics(results), - 'uniqueAudioMetrics': _metrics( - results - .where((item) => item['recordingId'] != 'human-smoke-003') - .toList(growable: false), - ), - }; - const outputPath = - 'test/eval/entity_search/human_voice_smoke_baseline_report.json'; - await File(outputPath) - .writeAsString(const JsonEncoder.withIndent(' ').convert(report)); - print(const JsonEncoder.withIndent(' ').convert(report)); - } finally { - speech.dispose(); - await db.close(); - } - }, timeout: const Timeout(Duration(minutes: 10))); -} - -Map _metrics(List> results) { - final wers = results - .map((item) => item['wer']) - .whereType() - .map((value) => value.toDouble()) - .toList(growable: false); - final cers = results - .map((item) => item['cer']) - .whereType() - .map((value) => value.toDouble()) - .toList(growable: false); - final canonical = results - .where((item) => item['canonicalEntityCorrect'] != null) - .toList(growable: false); - final canonicalOutcomes = results - .where((item) => item['canonicalOutcomeCorrect'] != null) - .toList(growable: false); - final entityMentions = results - .where((item) => item['entityMentionExact'] != null) - .toList(growable: false); - final outcomeCounts = {}; - for (final result in results) { - final outcome = result['finalOutcome'] as String; - outcomeCounts[outcome] = (outcomeCounts[outcome] ?? 0) + 1; - } - return { - 'files': results.length, - 'meanWer': wers.reduce((left, right) => left + right) / wers.length, - 'meanCer': cers.reduce((left, right) => left + right) / cers.length, - 'entityMentionExact': entityMentions - .where((item) => item['entityMentionExact'] == true) - .length, - 'entityMentionScorable': entityMentions.length, - 'finalCanonicalCorrect': canonical - .where((item) => item['canonicalEntityCorrect'] == true) - .length, - 'finalCanonicalScorable': canonical.length, - 'canonicalOutcomeCorrectIncludingClarification': canonicalOutcomes - .where((item) => item['canonicalOutcomeCorrect'] == true) - .length, - 'canonicalOutcomeScorable': canonicalOutcomes.length, - 'querySemanticsCorrect': results.where((item) { - final querySemantics = item['querySemantics']; - return querySemantics is Map && - querySemantics['semanticsCorrect'] == true; - }).length, - 'querySemanticsScorable': results.length, - 'outcomes': outcomeCounts, - }; -} diff --git a/test/eval/entity_search/index_coverage_audit_test.dart b/test/eval/entity_search/index_coverage_audit_test.dart deleted file mode 100644 index 4484fa0..0000000 --- a/test/eval/entity_search/index_coverage_audit_test.dart +++ /dev/null @@ -1,255 +0,0 @@ -// ignore_for_file: avoid_print -@Tags(['live-db', 'benchmark']) -library; - -import 'dart:convert'; -import 'dart:io'; - -import 'package:ai_rally_search/services/database_service.dart'; -import 'package:ai_rally_search/services/entity_search/entity_search_models.dart'; -import 'package:ai_rally_search/services/entity_search/mysql_entity_search_data_source.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/phonetic_matching_helper.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:flutter_test/flutter_test.dart'; - -void main() { - test('live-source versus index coverage audit', () async { - await dotenv.load(fileName: '.env'); - final db = DatabaseService(); - final entities = await MySqlEntitySearchDataSource(database: db) - .loadEntities(); - final profiles = await db.query(''' - SELECT 'driver' AS role, driver_id AS role_id, account_id, full_name, - country - FROM user_driver_profile - UNION ALL - SELECT 'co_driver' AS role, codriver_id AS role_id, account_id, full_name, - country - FROM user_codriver_profile; - '''); - - final indexedPeople = entities - .where((entity) => entity.entityType == SearchEntityType.person) - .toList(); - final nonNullAccountProfiles = profiles.where(_hasAccount).toList(); - final expectedAccountIds = nonNullAccountProfiles - .where(_hasName) - .map((row) => row['account_id'].toString()) - .toSet(); - final indexedAccountIds = indexedPeople - .map((entity) => entity.metadata['accountId']?.toString()) - .whereType() - .where((id) => id.isNotEmpty) - .toSet(); - final missingAccountIds = expectedAccountIds.difference(indexedAccountIds); - final missingPeople = >[]; - for (final accountId in missingAccountIds) { - final rows = profiles - .where((row) => row['account_id']?.toString() == accountId) - .toList(); - final reason = rows.every((row) => !_hasName(row)) - ? 'invalid/empty name' - : 'dedup bug'; - missingPeople.add({ - 'accountId': accountId, - 'reason': reason, - 'profiles': rows.map(_safeProfile).toList(), - }); - } - - final nullAccounts = profiles.where((row) => !_hasAccount(row)).toList(); - final nullLegitimate = nullAccounts.where(_hasName).toList(); - final nullDrivers = nullLegitimate - .where((row) => row['role'] == 'driver') - .toList(); - final nullCodrivers = nullLegitimate - .where((row) => row['role'] == 'co_driver') - .toList(); - final expectedPersonIds = { - for (final id in expectedAccountIds) 'person:account:$id', - for (final row in nullDrivers) 'person:driver:${row['role_id']}', - for (final row in nullCodrivers) 'person:codriver:${row['role_id']}', - }; - final indexedPersonIds = indexedPeople.map((e) => e.canonicalId).toSet(); - final missingPersonIds = expectedPersonIds.difference(indexedPersonIds); - final coverage = >{}; - coverage['PERSON'] = _coverage( - expectedPersonIds.length, - indexedPeople.length, - ); - for (final type in const [ - SearchEntityType.rally, - SearchEntityType.stage, - SearchEntityType.uploader, - ]) { - final sourceCount = await _sourceCount(db, type); - final indexed = entities.where((e) => e.entityType == type).length; - coverage[type.name.toUpperCase()] = _coverage(sourceCount, indexed); - } - - final namedTraces = >[]; - for (final expectedName in const ['Paweł Molgo', 'Shea Breen']) { - final normalized = PhoneticMatchingHelper.normalize(expectedName); - final exactProfiles = profiles.where((row) { - return PhoneticMatchingHelper.normalize( - row['full_name']?.toString() ?? '', - ) == - normalized; - }).toList(); - final accountIds = exactProfiles - .where(_hasAccount) - .map((row) => row['account_id'].toString()) - .toSet(); - final loaderEntities = indexedPeople.where((entity) { - if (accountIds.contains(entity.metadata['accountId']?.toString())) { - return true; - } - final names = [ - entity.canonicalName, - ...((entity.metadata['searchableNames'] as List?) ?? const []).map( - (name) => name.toString(), - ), - ]; - return names.any( - (name) => PhoneticMatchingHelper.normalize(name) == normalized, - ); - }).toList(); - namedTraces.add({ - 'expectedName': expectedName, - 'normalizedName': normalized, - 'profileRows': exactProfiles.map(_safeProfile).toList(), - 'accountIds': accountIds.toList(), - 'loaderOutput': loaderEntities - .map( - (entity) => { - 'canonicalId': entity.canonicalId, - 'canonicalName': entity.canonicalName, - 'metadata': entity.metadata, - }, - ) - .toList(), - 'absentReason': loaderEntities.isNotEmpty - ? null - : exactProfiles.isEmpty - ? 'No live driver/co-driver profile has this canonical full_name.' - : accountIds.isEmpty - ? 'Matching profile rows have NULL account_id and are excluded by loader policy.' - : loaderEntities.isEmpty - ? 'Unexpected loader/dedup omission.' - : null, - }); - } - - final report = { - 'personProfiles': { - 'totalDriverProfiles': profiles - .where((row) => row['role'] == 'driver') - .length, - 'totalCodriverProfiles': profiles - .where((row) => row['role'] == 'co_driver') - .length, - 'distinctNonNullAccountIds': nonNullAccountProfiles - .map((row) => row['account_id'].toString()) - .toSet() - .length, - 'profilesWithNullAccountId': nullAccounts.length, - 'namedProfilesWithNullAccountId': nullLegitimate.length, - 'distinctCanonicalPersonAccountsExpected': expectedAccountIds.length, - 'accountBackedIdentities': expectedAccountIds.length, - 'nullAccountDriverIdentities': nullDrivers.length, - 'nullAccountCodriverIdentities': nullCodrivers.length, - 'expectedTotalPersonCanonicalIdentities': expectedPersonIds.length, - 'actualPersonEntitiesIndexed': indexedPeople.length, - 'missingIdentityCount': missingPersonIds.length, - 'missingIdentityIds': missingPersonIds.toList()..sort(), - 'missingAccounts': missingPeople, - 'missingByReason': { - 'NULL account_id': 0, - 'loader filter': 0, - 'bad join': 0, - 'dedup bug': missingPeople - .where((row) => row['reason'] == 'dedup bug') - .length, - 'invalid/empty name': profiles - .where((row) => _hasAccount(row) && !_hasName(row)) - .length, - 'database inconsistency': 0, - 'other': 0, - }, - }, - 'coverage': coverage, - 'namedTraces': namedTraces, - }; - const path = 'test/eval/entity_search/index_coverage_audit_report.json'; - await File(path) - .writeAsString(const JsonEncoder.withIndent(' ').convert(report)); - print(const JsonEncoder.withIndent(' ').convert(report)); - expect(missingPeople.where((row) => row['reason'] == 'dedup bug'), isEmpty); - await db.close(); - }); -} - -bool _hasAccount(Map row) { - final value = row['account_id']?.toString().trim(); - return value != null && value.isNotEmpty && value != 'null'; -} - -bool _hasName(Map row) => - (row['full_name']?.toString().trim().isNotEmpty ?? false); - -Map _safeProfile(Map row) => { - 'role': row['role']?.toString(), - 'roleId': row['role_id']?.toString(), - 'accountId': _hasAccount(row) ? row['account_id']?.toString() : null, - 'fullName': row['full_name']?.toString(), - 'country': row['country']?.toString(), - 'excludedByLoader': - !_hasName(row) || (row['role_id']?.toString().trim().isEmpty ?? true), - 'exclusionReason': !_hasName(row) - ? 'invalid/empty name' - : (row['role_id']?.toString().trim().isEmpty ?? true) - ? 'invalid/empty role ID' - : null, - 'loaderIdentity': _hasAccount(row) - ? 'person:account:${row['account_id']}' - : row['role'] == 'driver' - ? 'person:driver:${row['role_id']}' - : 'person:codriver:${row['role_id']}', -}; - -Map _coverage(int source, int indexed) => { - 'sourceCanonicalCount': source, - 'indexedCanonicalCount': indexed, - 'missingCount': source - indexed, - 'coveragePercent': source == 0 ? 100.0 : indexed * 100 / source, -}; - -Future _sourceCount(DatabaseService db, SearchEntityType type) async { - final sql = switch (type) { - SearchEntityType.rally => - ''' - SELECT COUNT(DISTINCT event_id) AS count FROM rally_events - WHERE event_id IS NOT NULL AND event_name IS NOT NULL - AND TRIM(event_name) <> ''; - ''', - SearchEntityType.stage => - ''' - SELECT COUNT(DISTINCT stage_id) AS count FROM rally_stages - WHERE stage_id IS NOT NULL AND stage_name IS NOT NULL - AND TRIM(stage_name) <> ''; - ''', - SearchEntityType.uploader => - ''' - SELECT COUNT(DISTINCT fp.fan_id) AS count - FROM user_fan_profile fp - LEFT JOIN user_account ua ON ua.id = fp.account_id - WHERE fp.fan_id IS NOT NULL AND - COALESCE(NULLIF(TRIM(ua.user_name), ''), - NULLIF(TRIM(fp.full_name), ''), - NULLIF(TRIM(ua.email), '')) IS NOT NULL; - ''', - SearchEntityType.person => throw StateError('handled separately'), - }; - final rows = await db.query(sql); - return int.tryParse(rows.single['count'].toString()) ?? 0; -} diff --git a/test/eval/entity_search/known_real_device_entity_search_test.dart b/test/eval/entity_search/known_real_device_entity_search_test.dart deleted file mode 100644 index 1c1bad2..0000000 --- a/test/eval/entity_search/known_real_device_entity_search_test.dart +++ /dev/null @@ -1,224 +0,0 @@ -// ignore_for_file: avoid_print -@Tags(['live-db', 'benchmark']) -library; - -import 'dart:convert'; -import 'dart:io'; - -import 'package:ai_rally_search/services/database_service.dart'; -import 'package:ai_rally_search/models/search_intent.dart'; -import 'package:ai_rally_search/models/search_query.dart'; -import 'package:ai_rally_search/services/entity_search/controlled_fallback_entity_resolver.dart'; -import 'package:ai_rally_search/services/entity_search/entity_search_lookup_adapter.dart'; -import 'package:ai_rally_search/services/entity_search/entity_search_models.dart'; -import 'package:ai_rally_search/services/entity_search/in_memory_entity_search_service.dart'; -import 'package:ai_rally_search/services/entity_search/mysql_entity_search_data_source.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/database_entity_resolver.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/entity_lookup_repository.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/phonetic_matching_helper.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:flutter_test/flutter_test.dart'; - -void main() { - test('known real-device top five', () async { - await dotenv.load(fileName: '.env'); - final db = DatabaseService(); - final source = MySqlEntitySearchDataSource(database: db); - final allEntities = await source.loadEntities(); - final service = InMemoryEntitySearchService.fromEntities(allEntities); - final old = DatabaseEntityLookupRepository(dbService: db); - final legacyResolver = DatabaseEntityResolver(repository: old); - final integrated = ControlledFallbackEntityResolver( - legacyResolver: legacyResolver, - entitySearchResolver: DatabaseEntityResolver( - repository: EntitySearchLookupAdapter( - searchService: service, - cityFallback: old, - ), - ), - config: const EntitySearchFallbackConfig( - mode: EntitySearchFallbackMode.fallback, - ), - ); - const cases = <(String, SearchEntityType, String)>[ - ('aluksni', SearchEntityType.rally, 'aluksne'), - ('aluksnay', SearchEntityType.rally, 'aluksne'), - ('aluksney', SearchEntityType.rally, 'aluksne'), - ('alux new', SearchEntityType.rally, 'aluksne'), - ('a looks nay', SearchEntityType.rally, 'aluksne'), - ('eluksne', SearchEntityType.rally, 'aluksne'), - ('aluknse', SearchEntityType.rally, 'aluksne'), - ('pawel malgo', SearchEntityType.person, 'pawel molgo'), - ('shea brain', SearchEntityType.person, 'shea breen'), - ('donny gall', SearchEntityType.rally, 'donegal'), - ('kemel berg', SearchEntityType.stage, 'kemmelberg'), - ('dushniki', SearchEntityType.stage, 'duszniki'), - ]; - final results = >[]; - for (final item in cases) { - final request = EntitySearchRequest( - rawMention: item.$1, - entityType: item.$2, - limit: 5, - ); - final generated = service.candidateGenerator.generate(request); - final candidates = await service.search(request); - final generationStats = service.lastQueryStats!; - final targetIds = allEntities - .where( - (entity) => - entity.entityType == item.$2 && - PhoneticMatchingHelper.normalize(entity.canonicalName) - .contains(item.$3), - ) - .map((entity) => entity.canonicalId) - .toSet(); - final query = _query(item.$1, item.$2); - final legacy = await legacyResolver.resolve(query); - final finalResult = await integrated.resolveControlled( - query, - voice: true, - ); - final targetRank = candidates.indexWhere( - (candidate) => - PhoneticMatchingHelper.normalize(candidate.canonicalName) - .contains(item.$3), - ); - final resolved = finalResult.resolutions.values - .where((r) => r.isResolved) - .map((r) => r.resolvedCandidate?.canonicalName) - .whereType() - .firstOrNull; - results.add({ - 'input': item.$1, - 'type': item.$2.name, - 'legacyOutcome': _outcome(legacy), - 'newCandidateRank': targetRank < 0 ? null : targetRank + 1, - 'candidatePoolSize': generationStats.generatedCandidatePool, - 'fullUniverseSize': generationStats.fullUniverseSize, - 'fullScanEscape': generationStats.usedFullScanEscape, - 'canonicalTargetPresentInGeneratedPool': generated.canonicalIds.any( - targetIds.contains, - ), - 'finalResolverOutcome': _outcome(finalResult), - 'resolvedCanonicalName': resolved, - 'userVisibleBehavior': finalResult.requiresClarification - ? finalResult.clarificationQuestion - : resolved != null - ? 'execute canonical result' - : 'no match', - 'top5': candidates - .map( - (c) => { - 'id': c.canonicalId, - 'name': c.canonicalName, - 'score': c.score, - 'matchedSearchableName': c.metadata['matchedSearchableName'], - 'signals': c.signals.toMap(), - }, - ) - .toList(), - }); - } - const path = 'test/eval/entity_search/known_real_device_report.json'; - await File(path) - .writeAsString(const JsonEncoder.withIndent(' ').convert(results)); - final diagnostics = >[]; - for (final item in const <(String, String)>[ - ('pawel malgo', 'Paweł Molgo'), - ('shea brain', 'Shea Breen'), - ]) { - final expected = PhoneticMatchingHelper.normalize(item.$2); - final matchingAccounts = allEntities.where((entity) { - if (entity.entityType != SearchEntityType.person) return false; - final names = [ - entity.canonicalName, - ..._searchableNames(entity.metadata['searchableNames']), - ]; - return names.any( - (name) => PhoneticMatchingHelper.normalize(name) == expected, - ); - }).toList(); - final ranked = await service.search( - EntitySearchRequest( - rawMention: item.$1, - entityType: SearchEntityType.person, - limit: allEntities.length, - ), - ); - final accountIds = matchingAccounts.map((e) => e.canonicalId).toSet(); - final rank = ranked.indexWhere((c) => accountIds.contains(c.canonicalId)); - final matched = rank < 0 ? null : ranked[rank]; - diagnostics.add({ - 'input': item.$1, - 'expectedCanonicalName': item.$2, - 'queryNormalized': PhoneticMatchingHelper.normalize(item.$1), - 'entityExists': matchingAccounts.isNotEmpty, - 'matchingAccounts': matchingAccounts - .map( - (entity) => { - 'canonicalId': entity.canonicalId, - 'canonicalDisplayName': entity.canonicalName, - 'canonicalNormalized': PhoneticMatchingHelper.normalize( - entity.canonicalName, - ), - 'searchableNames': _searchableNames( - entity.metadata['searchableNames'], - ), - 'normalizedSearchableNames': _searchableNames( - entity.metadata['searchableNames'], - ).map(PhoneticMatchingHelper.normalize).toList(), - }, - ) - .toList(), - 'rank': rank < 0 ? null : rank + 1, - 'score': matched?.score, - 'signals': matched?.signals.toMap(), - 'strongestCompetitors': ranked - .take(5) - .map( - (candidate) => { - 'canonicalId': candidate.canonicalId, - 'canonicalName': candidate.canonicalName, - 'score': candidate.score, - 'signals': candidate.signals.toMap(), - }, - ) - .toList(), - }); - } - const diagnosticPath = - 'test/eval/entity_search/known_person_failure_diagnostic.json'; - await File(diagnosticPath) - .writeAsString(const JsonEncoder.withIndent(' ').convert(diagnostics)); - print(const JsonEncoder.withIndent(' ').convert(results)); - await db.close(); - }); -} - -List _searchableNames(Object? value) => switch (value) { - List values => values.map((value) => value.toString()).toList(), - Set values => values.map((value) => value.toString()).toList(), - String value => [value], - _ => const [], -}; - -SearchQuery _query(String mention, SearchEntityType type) => SearchQuery( - intent: type == SearchEntityType.person - ? SearchIntent.searchDriverVideos - : type == SearchEntityType.stage - ? SearchIntent.searchVideoActions - : SearchIntent.searchRallies, - driverNames: type == SearchEntityType.person ? [mention] : const [], - rallyNames: type == SearchEntityType.rally ? [mention] : const [], - stageNames: type == SearchEntityType.stage ? [mention] : const [], -); - -String _outcome(dynamic result) { - if (result.requiresClarification == true) return 'clarification'; - if (result.error != null) return 'no_match'; - if ((result.resolutions as Map).values.any((r) => r.isResolved == true)) { - return 'resolved'; - } - return 'no_match'; -} diff --git a/test/eval/entity_search/shared_fixture_benchmark_runner_test.dart b/test/eval/entity_search/shared_fixture_benchmark_runner_test.dart deleted file mode 100644 index 76e34b8..0000000 --- a/test/eval/entity_search/shared_fixture_benchmark_runner_test.dart +++ /dev/null @@ -1,586 +0,0 @@ -// ignore_for_file: avoid_print -@Tags(['live-db', 'benchmark']) -library; - -import 'dart:convert'; -import 'dart:io'; - -import 'package:ai_rally_search/models/entity_candidate.dart'; -import 'package:ai_rally_search/models/search_intent.dart'; -import 'package:ai_rally_search/models/search_query.dart'; -import 'package:ai_rally_search/services/database_service.dart'; -import 'package:ai_rally_search/services/entity_search/controlled_fallback_entity_resolver.dart'; -import 'package:ai_rally_search/services/entity_search/entity_candidate_generator.dart'; -import 'package:ai_rally_search/services/entity_search/entity_search_lookup_adapter.dart'; -import 'package:ai_rally_search/services/entity_search/entity_search_models.dart'; -import 'package:ai_rally_search/services/entity_search/in_memory_entity_search_service.dart'; -import 'package:ai_rally_search/services/entity_search/mysql_entity_search_data_source.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/database_entity_resolver.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/entity_lookup_repository.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/phonetic_matching_helper.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:flutter_test/flutter_test.dart'; - -void main() { - test('Dart Shared Fixture Benchmark Runner', () async { - await dotenv.load(fileName: '.env'); - final db = DatabaseService(); - final dataSource = MySqlEntitySearchDataSource(database: db); - final entities = await dataSource.loadEntities(); - - final indexedService = InMemoryEntitySearchService.fromEntities(entities); - final fullScanService = InMemoryEntitySearchService.fromEntities( - entities, - candidateGenerator: FullScanCandidateGenerator(), - ); - - final oldRepo = DatabaseEntityLookupRepository(dbService: db); - final metrics = EntitySearchFallbackMetrics(); - final adapter = EntitySearchLookupAdapter( - searchService: indexedService, - cityFallback: oldRepo, - metrics: metrics, - ); - final resolver = DatabaseEntityResolver(repository: adapter); - final fallbackResolver = ControlledFallbackEntityResolver( - legacyResolver: DatabaseEntityResolver(repository: oldRepo), - entitySearchResolver: resolver, - config: const EntitySearchFallbackConfig( - mode: EntitySearchFallbackMode.fallback, - ), - metrics: metrics, - ); - - SearchEntityType parseEntityType(String typeStr) { - switch (typeStr.toLowerCase()) { - case 'rally': - return SearchEntityType.rally; - case 'person': - case 'driver': - case 'co_driver': - return SearchEntityType.person; - case 'stage': - return SearchEntityType.stage; - case 'uploader': - return SearchEntityType.uploader; - default: - return SearchEntityType.person; - } - } - - PersonRole parsePersonRole(String? roleStr) { - if (roleStr == null) return PersonRole.any; - switch (roleStr.toLowerCase()) { - case 'driver': - return PersonRole.driver; - case 'co_driver': - case 'codriver': - return PersonRole.coDriver; - default: - return PersonRole.any; - } - } - - // ========================================================================= - // 1. RUN 803 SUITE - // ========================================================================= - final file803 = File('test/eval/entity_search/frozen_803_cases.json'); - expect(file803.existsSync(), isTrue); - final cases803 = json.decode(file803.readAsStringSync()) as List; - expect(cases803.length, 803); - - final results803 = >[]; - var r1Count803 = 0, r5Count803 = 0, r10Count803 = 0; - var reciprocalSum803 = 0.0; - var escapes803 = 0; - - for (final rawCase in cases803) { - final c = rawCase as Map; - final caseId = c['caseId'] as String; - final targetId = c['targetCanonicalId'] as String; - final targetName = c['targetCanonicalName'] as String; - final type = parseEntityType(c['entityType'] as String); - final role = parsePersonRole(c['personRole'] as String?); - final input = c['input'] as String; - - final req = EntitySearchRequest( - rawMention: input, - entityType: type, - personRole: role, - limit: 10, - ); - - final generated = indexedService.candidateGenerator.generate(req); - final candidates = await indexedService.search(req); - final stats = indexedService.lastQueryStats; - - final poolSize = generated.canonicalIds.length; - final targetPresent = generated.canonicalIds.contains(targetId); - final isEscape = stats?.usedFullScanEscape ?? false; - if (isEscape) escapes803++; - - int? targetRank; - for (var i = 0; i < candidates.length; i++) { - if (candidates[i].canonicalId == targetId) { - targetRank = i + 1; - break; - } - } - - if (targetRank == 1) { - r1Count803++; - r5Count803++; - r10Count803++; - reciprocalSum803 += 1.0; - } else if (targetRank != null && targetRank <= 5) { - r5Count803++; - r10Count803++; - reciprocalSum803 += 1.0 / targetRank; - } else if (targetRank != null && targetRank <= 10) { - r10Count803++; - reciprocalSum803 += 1.0 / targetRank; - } - - final topScore = candidates.isNotEmpty ? candidates.first.score : 0.0; - final secondScore = candidates.length > 1 ? candidates[1].score : 0.0; - final scoreGap = candidates.isNotEmpty ? topScore - secondScore : 0.0; - - // Deterministic Resolver evaluation - final query = type == SearchEntityType.person - ? SearchQuery( - intent: SearchIntent.searchDriverVideos, - driverNames: [input], - personRole: role, - ) - : type == SearchEntityType.rally - ? SearchQuery( - intent: SearchIntent.searchRallies, - rallyNames: [input], - ) - : SearchQuery( - intent: SearchIntent.searchVideoActions, - stageNames: [input], - ); - - final resResult = await resolver.resolve(query); - final primaryRes = resResult.resolutions.values.firstOrNull; - - String resolverOutcome; - String? selectedCanonicalId; - String resolverReason; - - if (primaryRes == null || !primaryRes.isResolved && !primaryRes.isAmbiguous) { - resolverOutcome = 'NO_MATCH'; - resolverReason = primaryRes?.strategy ?? 'no_resolution'; - } else if (primaryRes.isAmbiguous || resResult.requiresClarification) { - resolverOutcome = 'CLARIFICATION'; - resolverReason = primaryRes.strategy; - } else { - resolverOutcome = 'RESOLVED'; - selectedCanonicalId = primaryRes.resolvedCandidate?.id; - resolverReason = primaryRes.strategy; - } - - results803.add({ - 'caseId': caseId, - 'expectedCanonicalId': targetId, - 'expectedCanonicalName': targetName, - 'entityType': c['entityType'], - 'personRole': c['personRole'], - 'input': input, - 'candidatePoolSize': poolSize, - 'targetPresent': targetPresent, - 'targetRank': targetRank, - 'topCandidates': candidates.map((cand) => cand.canonicalId).toList(), - 'resolverOutcome': resolverOutcome, - 'selectedCanonicalId': selectedCanonicalId, - 'resolverReason': resolverReason, - 'topScore': topScore, - 'scoreGap': scoreGap, - 'fullScanEscape': isEscape, - 'escapeReason': isEscape ? 'insufficient_candidates' : null, - }); - } - - final summary803 = { - 'cases': cases803.length, - 'recallAt1': r1Count803 / cases803.length, - 'recallAt5': r5Count803 / cases803.length, - 'recallAt10': r10Count803 / cases803.length, - 'mrr': reciprocalSum803 / cases803.length, - 'fullScanEscapes': escapes803, - 'results': results803, - }; - File('test/eval/entity_search/dart_803_results.json').writeAsStringSync( - const JsonEncoder.withIndent(' ').convert(summary803), - ); - print('Dart 803: R@1=${summary803['recallAt1']} MRR=${summary803['mrr']} escapes=$escapes803'); - - // ========================================================================= - // 2. RUN PERSON_FROZEN_1101 SUITE - // ========================================================================= - final file1101 = File('test/eval/entity_search/frozen_1101_person_cases.json'); - expect(file1101.existsSync(), isTrue); - final cases1101 = json.decode(file1101.readAsStringSync()) as List; - expect(cases1101.length, 1101); - - final results1101 = >[]; - var r1Count1101 = 0, r5Count1101 = 0, r10Count1101 = 0; - var reciprocalSum1101 = 0.0; - var escapes1101 = 0; - - final groupMetrics = >{ - 'ACCOUNT_BACKED': {'cases': 0, 'r1': 0, 'r5': 0, 'r10': 0, 'mrr': 0.0}, - 'NULL_DRIVER': {'cases': 0, 'r1': 0, 'r5': 0, 'r10': 0, 'mrr': 0.0}, - 'NULL_CODRIVER': {'cases': 0, 'r1': 0, 'r5': 0, 'r10': 0, 'mrr': 0.0}, - }; - - for (final rawCase in cases1101) { - final c = rawCase as Map; - final caseId = c['caseId'] as String; - final group = c['group'] as String; - final targetId = c['targetCanonicalId'] as String; - final targetName = c['targetCanonicalName'] as String; - final role = parsePersonRole(c['personRole'] as String?); - final input = c['input'] as String; - - final req = EntitySearchRequest( - rawMention: input, - entityType: SearchEntityType.person, - personRole: role, - limit: 10, - ); - - final generated = indexedService.candidateGenerator.generate(req); - final candidates = await indexedService.search(req); - final stats = indexedService.lastQueryStats; - - final poolSize = generated.canonicalIds.length; - final targetPresent = generated.canonicalIds.contains(targetId); - final isEscape = stats?.usedFullScanEscape ?? false; - if (isEscape) escapes1101++; - - int? targetRank; - for (var i = 0; i < candidates.length; i++) { - if (candidates[i].canonicalId == targetId) { - targetRank = i + 1; - break; - } - } - - final gm = groupMetrics[group]!; - gm['cases'] = (gm['cases'] as int) + 1; - - if (targetRank == 1) { - r1Count1101++; - r5Count1101++; - r10Count1101++; - reciprocalSum1101 += 1.0; - gm['r1'] = (gm['r1'] as int) + 1; - gm['r5'] = (gm['r5'] as int) + 1; - gm['r10'] = (gm['r10'] as int) + 1; - gm['mrr'] = (gm['mrr'] as double) + 1.0; - } else if (targetRank != null && targetRank <= 5) { - r5Count1101++; - r10Count1101++; - reciprocalSum1101 += 1.0 / targetRank; - gm['r5'] = (gm['r5'] as int) + 1; - gm['r10'] = (gm['r10'] as int) + 1; - gm['mrr'] = (gm['mrr'] as double) + 1.0 / targetRank; - } else if (targetRank != null && targetRank <= 10) { - r10Count1101++; - reciprocalSum1101 += 1.0 / targetRank; - gm['r10'] = (gm['r10'] as int) + 1; - gm['mrr'] = (gm['mrr'] as double) + 1.0 / targetRank; - } - - final topScore = candidates.isNotEmpty ? candidates.first.score : 0.0; - final secondScore = candidates.length > 1 ? candidates[1].score : 0.0; - final scoreGap = candidates.isNotEmpty ? topScore - secondScore : 0.0; - - final query = SearchQuery( - intent: SearchIntent.searchDriverVideos, - driverNames: [input], - personRole: role, - ); - - final resResult = await resolver.resolve(query); - final primaryRes = resResult.resolutions.values.firstOrNull; - - String resolverOutcome; - String? selectedCanonicalId; - String resolverReason; - - if (primaryRes == null || !primaryRes.isResolved && !primaryRes.isAmbiguous) { - resolverOutcome = 'NO_MATCH'; - resolverReason = primaryRes?.strategy ?? 'no_resolution'; - } else if (primaryRes.isAmbiguous || resResult.requiresClarification) { - resolverOutcome = 'CLARIFICATION'; - resolverReason = primaryRes.strategy; - } else { - resolverOutcome = 'RESOLVED'; - selectedCanonicalId = primaryRes.resolvedCandidate?.id; - resolverReason = primaryRes.strategy; - } - - results1101.add({ - 'caseId': caseId, - 'group': group, - 'expectedCanonicalId': targetId, - 'expectedCanonicalName': targetName, - 'entityType': 'person', - 'personRole': c['personRole'], - 'input': input, - 'candidatePoolSize': poolSize, - 'targetPresent': targetPresent, - 'targetRank': targetRank, - 'topCandidates': candidates.map((cand) => cand.canonicalId).toList(), - 'resolverOutcome': resolverOutcome, - 'selectedCanonicalId': selectedCanonicalId, - 'resolverReason': resolverReason, - 'topScore': topScore, - 'scoreGap': scoreGap, - 'fullScanEscape': isEscape, - 'escapeReason': isEscape ? 'insufficient_candidates' : null, - }); - } - - final summary1101 = { - 'cases': cases1101.length, - 'recallAt1': r1Count1101 / cases1101.length, - 'recallAt5': r5Count1101 / cases1101.length, - 'recallAt10': r10Count1101 / cases1101.length, - 'mrr': reciprocalSum1101 / cases1101.length, - 'byGroup': { - for (final entry in groupMetrics.entries) - entry.key: { - 'cases': entry.value['cases'], - 'recallAt1': (entry.value['r1'] as int) / (entry.value['cases'] as int), - 'recallAt5': (entry.value['r5'] as int) / (entry.value['cases'] as int), - 'recallAt10': (entry.value['r10'] as int) / (entry.value['cases'] as int), - 'mrr': (entry.value['mrr'] as double) / (entry.value['cases'] as int), - }, - }, - 'results': results1101, - }; - File('test/eval/entity_search/dart_1101_person_results.json').writeAsStringSync( - const JsonEncoder.withIndent(' ').convert(summary1101), - ); - print('Dart 1101: R@1=${summary1101['recallAt1']} MRR=${summary1101['mrr']}'); - - // ========================================================================= - // 3. RUN 168 SAFETY SUITE - // ========================================================================= - final file168 = File('test/eval/entity_search/frozen_168_safety_cases.json'); - expect(file168.existsSync(), isTrue); - final cases168 = json.decode(file168.readAsStringSync()) as List; - expect(cases168.length, 168); - - final results168 = >[]; - var correctConfident168 = 0; - var wrongPositiveConfident168 = 0; - var positiveClarification168 = 0; - var positiveNoMatch168 = 0; - var negativeWrongConfident168 = 0; - var negativeClarification168 = 0; - var negativeRejection168 = 0; - - for (final rawCase in cases168) { - final c = rawCase as Map; - final caseId = c['caseId'] as String; - final category = c['category'] as String; - final input = c['input'] as String; - final expectedName = c['expectedCanonicalName'] as String?; - final type = parseEntityType(c['entityType'] as String); - final role = parsePersonRole(c['personRole'] as String?); - - final query = type == SearchEntityType.person - ? SearchQuery( - intent: SearchIntent.searchDriverVideos, - driverNames: [input], - personRole: role, - ) - : type == SearchEntityType.rally - ? SearchQuery( - intent: SearchIntent.searchRallies, - rallyNames: [input], - ) - : SearchQuery( - intent: SearchIntent.searchVideoActions, - stageNames: [input], - ); - - final result = await fallbackResolver.resolve(query); - final primary = result.resolutions.values.firstOrNull; - final resolvedName = primary?.resolvedCandidate?.canonicalName; - final isClarification = result.requiresClarification || (primary?.isAmbiguous ?? false); - final isResolved = primary != null && primary.isResolved && resolvedName != null; - - String outcome; - if (isResolved) { - outcome = 'RESOLVED'; - } else if (isClarification) { - outcome = 'CLARIFICATION'; - } else { - outcome = 'NO_MATCH'; - } - - if (category == 'positive') { - final sameTarget = resolvedName != null && - expectedName != null && - (PhoneticMatchingHelper.normalize(resolvedName) == PhoneticMatchingHelper.normalize(expectedName) || - resolvedName.contains(expectedName) || - expectedName.contains(resolvedName)); - - if (isResolved && sameTarget) { - correctConfident168++; - } else if (isResolved) { - wrongPositiveConfident168++; - } else if (isClarification) { - positiveClarification168++; - } else { - positiveNoMatch168++; - } - } else { - if (isResolved) { - negativeWrongConfident168++; - } else if (isClarification) { - negativeClarification168++; - } else { - negativeRejection168++; - } - } - - results168.add({ - 'caseId': caseId, - 'category': category, - 'input': input, - 'expectedCanonicalName': expectedName, - 'entityType': c['entityType'], - 'personRole': c['personRole'], - 'resolverOutcome': outcome, - 'selectedCanonicalName': resolvedName, - 'selectedCanonicalId': primary?.resolvedCandidate?.id, - 'resolverReason': primary?.strategy, - 'requiresClarification': isClarification, - }); - } - - final summary168 = { - 'totalQueries': cases168.length, - 'positive': { - 'queries': 62, - 'correctConfident': correctConfident168, - 'wrongConfident': wrongPositiveConfident168, - 'clarification': positiveClarification168, - 'noMatch': positiveNoMatch168, - }, - 'negativeConfusable': { - 'queries': 106, - 'wrongConfident': negativeWrongConfident168, - 'clarification': negativeClarification168, - 'rejection': negativeRejection168, - }, - 'falseConfidentAutoResolution': wrongPositiveConfident168 + negativeWrongConfident168, - 'results': results168, - }; - File('test/eval/entity_search/dart_168_safety_results.json').writeAsStringSync( - const JsonEncoder.withIndent(' ').convert(summary168), - ); - print('Dart 168: falseConfident=${summary168['falseConfidentAutoResolution']}'); - - // ========================================================================= - // 4. RUN KNOWN TRANSCRIPTS SUITE - // ========================================================================= - final fileTranscripts = File('test/eval/entity_search/frozen_known_transcripts_cases.json'); - expect(fileTranscripts.existsSync(), isTrue); - final casesTranscripts = json.decode(fileTranscripts.readAsStringSync()) as List; - expect(casesTranscripts.length, 12); - - final resultsTranscripts = >[]; - for (final rawCase in casesTranscripts) { - final c = rawCase as Map; - final caseId = c['caseId'] as String; - final input = c['input'] as String; - final expectedName = c['expectedCanonicalName'] as String; - final type = parseEntityType(c['entityType'] as String); - final role = parsePersonRole(c['personRole'] as String?); - - final req = EntitySearchRequest( - rawMention: input, - entityType: type, - personRole: role, - limit: 10, - ); - - final generated = indexedService.candidateGenerator.generate(req); - final candidates = await indexedService.search(req); - final stats = indexedService.lastQueryStats; - - final poolSize = generated.canonicalIds.length; - final isEscape = stats?.usedFullScanEscape ?? false; - - int? targetRank; - for (var i = 0; i < candidates.length; i++) { - if (candidates[i].canonicalName == expectedName || - PhoneticMatchingHelper.normalize(candidates[i].canonicalName) == PhoneticMatchingHelper.normalize(expectedName)) { - targetRank = i + 1; - break; - } - } - - final query = type == SearchEntityType.person - ? SearchQuery( - intent: SearchIntent.searchDriverVideos, - driverNames: [input], - personRole: role, - ) - : type == SearchEntityType.rally - ? SearchQuery( - intent: SearchIntent.searchRallies, - rallyNames: [input], - ) - : SearchQuery( - intent: SearchIntent.searchVideoActions, - stageNames: [input], - ); - - final resResult = await resolver.resolve(query); - final primaryRes = resResult.resolutions.values.firstOrNull; - - String resolverOutcome; - String? selectedName; - if (primaryRes != null && primaryRes.isResolved && primaryRes.resolvedCandidate != null) { - resolverOutcome = 'RESOLVED'; - selectedName = primaryRes.resolvedCandidate!.canonicalName; - } else if (resResult.requiresClarification || (primaryRes?.isAmbiguous ?? false)) { - resolverOutcome = 'CLARIFICATION'; - } else { - resolverOutcome = 'NO_MATCH'; - } - - resultsTranscripts.add({ - 'caseId': caseId, - 'input': input, - 'expectedCanonicalName': expectedName, - 'entityType': c['entityType'], - 'personRole': c['personRole'], - 'candidatePoolSize': poolSize, - 'targetPresent': targetRank != null, - 'targetRank': targetRank, - 'topCandidateName': candidates.isNotEmpty ? candidates.first.canonicalName : null, - 'topCandidateScore': candidates.isNotEmpty ? candidates.first.score : 0.0, - 'resolverOutcome': resolverOutcome, - 'resolverReason': primaryRes?.strategy, - 'selectedCanonicalName': selectedName, - 'fullScanEscape': isEscape, - }); - } - - File('test/eval/entity_search/dart_known_transcripts_results.json').writeAsStringSync( - const JsonEncoder.withIndent(' ').convert(resultsTranscripts), - ); - print('Dart Known Transcripts complete: 12 cases.'); - await db.close(); - }, timeout: const Timeout(Duration(minutes: 15))); -} diff --git a/test/eval/entity_search/synthetic_stt_biasing_benchmark_test.dart b/test/eval/entity_search/synthetic_stt_biasing_benchmark_test.dart deleted file mode 100644 index 10f97c2..0000000 --- a/test/eval/entity_search/synthetic_stt_biasing_benchmark_test.dart +++ /dev/null @@ -1,425 +0,0 @@ -// ignore_for_file: avoid_print -@Tags(['live-db', 'live-api', 'benchmark']) -library; - -import 'dart:convert'; -import 'dart:io'; -import 'dart:math'; -import 'dart:typed_data'; - -import 'package:ai_rally_search/services/database_service.dart'; -import 'package:ai_rally_search/services/entity_search/controlled_fallback_entity_resolver.dart'; -import 'package:ai_rally_search/services/entity_search/entity_search_lookup_adapter.dart'; -import 'package:ai_rally_search/services/entity_search/entity_search_models.dart'; -import 'package:ai_rally_search/services/entity_search/in_memory_entity_search_service.dart'; -import 'package:ai_rally_search/services/entity_search/mysql_entity_search_data_source.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/database_entity_resolver.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/entity_lookup_repository.dart'; -import 'package:ai_rally_search/services/llm/llm_query_parser_factory.dart'; -import 'package:ai_rally_search/services/speech/openai_speech_to_text_service.dart'; -import 'package:ai_rally_search/services/speech/speech_config.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:flutter_test/flutter_test.dart'; - -import 'synthetic_stt_audio_fixture_generator.dart'; -import 'synthetic_stt_biasing_corpus.dart'; -import 'synthetic_stt_biasing_evaluator.dart'; - -void main() { - test('ES-6A synthetic STT biasing benchmark', () async { - await dotenv.load(fileName: '.env'); - final apiKey = dotenv.env['OPENAI_API_KEY']; - expect(apiKey, isNotNull); - expect(apiKey, isNotEmpty); - final db = DatabaseService(); - final entities = await MySqlEntitySearchDataSource(database: db) - .loadEntities(); - final corpus = SyntheticSttCorpusBuilder().build(entities); - expect(corpus.entities.length, 140); - expect(corpus.utterances.length, 280); - - const fixturePath = 'test/eval/audio/es6a'; - final audioDirectory = Directory(fixturePath); - final audioStats = await SyntheticSttAudioFixtureGenerator(audioDirectory) - .generate(corpus); - - final negatives = _negativeCorpus(entities, corpus); - await SyntheticSttAudioFixtureGenerator(audioDirectory).generate( - SyntheticSttCorpus( - entities: negatives.map((item) => item.target).toList(), - utterances: negatives, - ), - ); - - final service = InMemoryEntitySearchService.fromEntities(entities); - final legacy = DatabaseEntityLookupRepository(dbService: db); - final resolver = ControlledFallbackEntityResolver( - legacyResolver: DatabaseEntityResolver(repository: legacy), - entitySearchResolver: DatabaseEntityResolver( - repository: EntitySearchLookupAdapter( - searchService: service, - cityFallback: legacy, - ), - ), - config: const EntitySearchFallbackConfig( - mode: EntitySearchFallbackMode.fallback, - ), - ); - final speech = OpenAiSpeechToTextService( - config: SpeechConfig( - providerType: SpeechProviderType.openAiDirectDev, - endpointUrl: 'https://api.openai.com/v1/audio/transcriptions', - apiKey: apiKey, - model: 'gpt-transcribe', - timeout: const Duration(seconds: 45), - ), - ); - expect(speech.transcriptionCapabilities.keywordHints, isTrue); - final evaluator = SyntheticSttBiasingEvaluator( - speech: speech, - parser: LlmQueryParserFactory.create(), - resolver: resolver, - entitySearch: service, - cacheFile: File('test/eval/entity_search/es6a_transcript_cache.json'), - ); - final configuredMax = int.tryParse( - Platform.environment['ES6A_MAX_AUDIO_FILES'] ?? '', - ); - final results = await evaluator.evaluate( - corpus.utterances, - audioDirectory, - maxAudioFiles: configuredMax, - onProgress: (done, total) => print('ES-6A audio $done/$total'), - ); - final biasNames = corpus.entities.map((e) => e.canonicalName).toList(); - final personBiasNames = [ - ...entities - .where( - (entity) => - entity.entityType == SearchEntityType.person && - entity.canonicalName == 'Paweł Molgo', - ) - .map((entity) => entity.canonicalName), - ...corpus.entities - .where((entity) => entity.entityType == SearchEntityType.person) - .map((entity) => entity.canonicalName), - ].where((name) => name != 'Josh Moffett').toSet().take(10).toList(); - final negativeCases = [ - NegativeBiasCase( - id: 'negative_all_rallies', - spokenText: negatives[0].text, - audioFile: File('$fixturePath/${negatives[0].id}_clean.wav'), - biasVocabulary: biasNames - .where( - (name) => - corpus.entities - .firstWhere((e) => e.canonicalName == name) - .entityType == - SearchEntityType.rally, - ) - .take(10) - .toList(), - ), - NegativeBiasCase( - id: 'negative_another_rally', - spokenText: negatives[1].text, - audioFile: File('$fixturePath/${negatives[1].id}_clean.wav'), - biasVocabulary: biasNames.take(10).toList(), - ), - NegativeBiasCase( - id: 'negative_wrong_person', - spokenText: negatives[2].text, - audioFile: File('$fixturePath/${negatives[2].id}_clean.wav'), - biasVocabulary: personBiasNames, - expectedSpokenCanonicalName: 'Josh Moffett', - ), - ]; - final negativeResults = await evaluator.evaluateNegativeBias(negativeCases); - final summary = _summarize(results, negativeResults); - final actualSttAudioSeconds = _actualSttAudioSeconds( - results, - audioDirectory, - negativeCases, - negativeResults, - ); - final report = { - 'phase': 'ES-6A', - 'humanVoiceBenchmark': 'BLOCKED', - 'syntheticOnly': true, - 'seed': syntheticSttBiasingSeed, - 'corpus': { - 'entities': { - for (final type in SearchEntityType.values) - type.name: corpus.entities - .where((e) => e.entityType == type) - .length, - }, - 'entityNames': { - for (final type in SearchEntityType.values) - type.name: corpus.entities - .where((e) => e.entityType == type) - .map((e) => e.canonicalName) - .toList(), - }, - 'utterances': corpus.utterances.length, - 'audioFilesExpected': corpus.utterances.length * 2, - 'audioConditions': ['clean', 'noisy'], - 'voices': ['Samantha', 'Daniel'], - 'speakingRatesWpm': [165, 180, 205, 210], - 'noisyPerturbations': [ - 'deterministic low background noise', - 'volume reduction', - 'mild dynamic compression', - 'leading/trailing silence', - ], - 'manifest': corpus.utterances.map((e) => e.toJson()).toList(), - }, - 'provider': { - 'stt': 'OpenAI gpt-transcribe', - 'capabilitiesUsed': ['prompt', 'keywords[]', 'languages[]'], - 'tts': 'macOS say local system TTS', - }, - 'audioGeneration': audioStats.toJson(), - 'results': summary, - 'negativeBias': { - 'cases': negativeResults, - 'biasInducedEntityErrors': negativeResults - .where((item) => item['biasInducedEntityError'] == true) - .length, - }, - 'cost': { - 'ttsApiCalls': 0, - 'localTtsSynthesesForFullCorpus': corpus.utterances.length * 2, - 'syntheticAudioDurationSeconds': audioStats.totalDurationMs / 1000, - 'baselineSttCalls': - results - .where((e) => e.strategy == SyntheticSttStrategy.baseline) - .length + - negativeCases.length, - 'staticContextSttCalls': results - .where((e) => e.strategy == SyntheticSttStrategy.staticContext) - .length, - 'dynamicSecondPassSttCalls': - results.where((e) => e.secondPassTriggered).length + - negativeResults.length, - 'sttCalls': - _actualSttCalls(results) + - negativeCases.length + - negativeResults.length, - 'measuredSttAudioSeconds': actualSttAudioSeconds, - 'gptTranscribePricePerMinuteUsd': 0.0045, - 'estimatedExperimentalSttCostUsd': actualSttAudioSeconds / 60 * 0.0045, - 'productionCostExtrapolation': null, - }, - 'historicalTranscriptRegression': { - 'audioBenchmark': false, - 'inputs': [ - 'aluksni', - 'aluksnay', - 'aluksney', - 'alux new', - 'a looks nay', - 'eluksne', - 'aluknse', - 'pawel malgo', - 'shea brain', - 'donny gall', - 'kemel berg', - 'dushniki', - ], - 'report': 'test/eval/entity_search/known_real_device_report.json', - }, - 'limitations': [ - 'Synthetic TTS is not human speech or accent validation.', - 'Dynamic-biased STT output is causally influenced by Entity Search and is not independent confirmation.', - if (configuredMax != null) - 'Execution was explicitly limited to $configuredMax audio files.', - ], - 'details': results.map((e) => e.toJson()).toList(), - }; - const reportPath = - 'test/eval/entity_search/synthetic_stt_biasing_report.json'; - await File(reportPath) - .writeAsString(const JsonEncoder.withIndent(' ').convert(report)); - print(const JsonEncoder.withIndent(' ').convert(summary)); - speech.dispose(); - await db.close(); - }, timeout: const Timeout(Duration(hours: 3))); -} - -List _negativeCorpus( - List source, - SyntheticSttCorpus corpus, -) { - final rally = corpus.entities.firstWhere( - (e) => e.entityType == SearchEntityType.rally, - ); - final person = corpus.entities.firstWhere( - (e) => e.entityType == SearchEntityType.person, - ); - return [ - SyntheticSttUtterance( - id: 'negative_all_rallies', - target: rally, - text: 'show all rallies', - templateIndex: 0, - ), - SyntheticSttUtterance( - id: 'negative_another_rally', - target: rally, - text: 'show another rally', - templateIndex: 1, - ), - SyntheticSttUtterance( - id: 'negative_josh_moffett', - target: person, - text: 'show Josh Moffett', - templateIndex: 0, - ), - ]; -} - -Map _summarize( - List results, - List> negatives, -) { - final baseline = { - for (final result in results.where( - (r) => r.strategy == SyntheticSttStrategy.baseline, - )) - '${result.sampleId}|${result.audioCondition}': result, - }; - return { - for (final strategy in SyntheticSttStrategy.values) - strategy.name: _strategySummary( - results.where((r) => r.strategy == strategy).toList(), - baseline, - negatives - .where( - (item) => - item['topK'] == - switch (strategy) { - SyntheticSttStrategy.dynamicTop3 => 3, - SyntheticSttStrategy.dynamicTop5 => 5, - SyntheticSttStrategy.dynamicTop10 => 10, - _ => -1, - }, - ) - .toList(), - ), - }; -} - -Map _strategySummary( - List values, - Map baseline, - List> negatives, -) { - if (values.isEmpty) return {'cases': 0}; - final totalLatency = values.map((e) => e.totalLatencyMs).toList()..sort(); - var positiveBiasErrors = 0; - for (final value in values) { - final original = baseline['${value.sampleId}|${value.audioCondition}']; - if (original != null && - !original.wrongConfident && - ((original.canonicalAt1 && !value.canonicalAt1) || - value.wrongConfident)) { - positiveBiasErrors++; - } - } - return { - 'cases': values.length, - 'canonicalAccuracy': - values.where((e) => e.canonicalAt1).length / values.length, - 'canonicalRecallAt1': - values.where((e) => e.canonicalAt1).length / values.length, - 'correctConfident': values.where((e) => e.correctConfident).length, - 'clarification': values.where((e) => e.clarification).length, - 'noMatch': values.where((e) => e.noMatch).length, - 'wrongConfident': values.where((e) => e.wrongConfident).length, - 'secondPassTriggerRate': - values.where((e) => e.secondPassTriggered).length / values.length, - 'averageSttCallsPerQuery': - values.map((e) => e.sttCalls).reduce((a, b) => a + b) / values.length, - 'averageSttLatencyMs': - values.map((e) => e.sttLatencyMs).reduce((a, b) => a + b) / - values.length, - 'averageTotalLatencyMs': - totalLatency.reduce((a, b) => a + b) / values.length, - 'p95TotalLatencyMs': - totalLatency[min( - totalLatency.length - 1, - (totalLatency.length * .95).floor(), - )], - 'wer': values.map((e) => e.wer).reduce((a, b) => a + b) / values.length, - 'cer': values.map((e) => e.cer).reduce((a, b) => a + b) / values.length, - 'biasInducedEntityErrors': - positiveBiasErrors + - negatives.where((e) => e['biasInducedEntityError'] == true).length, - 'cleanCanonicalAccuracy': _conditionAccuracy(values, 'clean'), - 'noisyCanonicalAccuracy': _conditionAccuracy(values, 'noisy'), - }; -} - -double _conditionAccuracy( - List values, - String condition, -) { - final selected = values.where((e) => e.audioCondition == condition).toList(); - return selected.isEmpty - ? 0 - : selected.where((e) => e.canonicalAt1).length / selected.length; -} - -int _actualSttCalls(List results) { - final baseline = results - .where((e) => e.strategy == SyntheticSttStrategy.baseline) - .length; - final staticCalls = results - .where((e) => e.strategy == SyntheticSttStrategy.staticContext) - .length; - final second = results.where((e) => e.secondPassTriggered).length; - return baseline + staticCalls + second; -} - -double _actualSttAudioSeconds( - List results, - Directory directory, - List negativeCases, - List> negativeResults, -) { - var seconds = 0.0; - for (final baseline in results.where( - (e) => e.strategy == SyntheticSttStrategy.baseline, - )) { - final file = File( - '${directory.path}/${baseline.sampleId}_${baseline.audioCondition}.wav', - ); - final duration = _wavSeconds(file); - seconds += duration * 2; - final dynamicCalls = results - .where( - (e) => - e.sampleId == baseline.sampleId && - e.audioCondition == baseline.audioCondition && - e.secondPassTriggered, - ) - .length; - seconds += duration * dynamicCalls; - } - for (final negative in negativeCases) { - final dynamicCalls = negativeResults - .where((result) => result['id'] == negative.id) - .length; - seconds += _wavSeconds(negative.audioFile) * (1 + dynamicCalls); - } - return seconds; -} - -double _wavSeconds(File file) { - final bytes = file.readAsBytesSync(); - final data = ByteData.sublistView(bytes); - final byteRate = data.getUint32(28, Endian.little); - final dataBytes = data.getUint32(40, Endian.little); - return dataBytes / byteRate; -} diff --git a/test/eval/live_db_generalization_test.dart b/test/eval/live_db_generalization_test.dart deleted file mode 100644 index c061c43..0000000 --- a/test/eval/live_db_generalization_test.dart +++ /dev/null @@ -1,165 +0,0 @@ -import 'package:flutter_test/flutter_test.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:ai_rally_search/services/database_service.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/entity_lookup_repository.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/database_entity_resolver.dart'; -import 'package:ai_rally_search/models/search_query.dart'; -import 'package:ai_rally_search/models/search_intent.dart'; - -void main() { - group('Phase 5B.2 — Live Database Generalization Integration Test Suite', () { - late DatabaseService dbService; - late DatabaseEntityLookupRepository lookupRepo; - late DatabaseEntityResolver resolver; - - setUpAll(() async { - await dotenv.load(fileName: '.env'); - dbService = DatabaseService(); - lookupRepo = DatabaseEntityLookupRepository(dbService: dbService); - resolver = DatabaseEntityResolver(repository: lookupRepo); - }); - - test('Live DB Entity 1: "Keith Cronan" resolves to Keith Cronin (49d7ab8d-6d05-4015-9af1-5195f33b647f)', () async { - final query = const SearchQuery( - intent: SearchIntent.searchDriverVideos, - driverName: 'Keith Cronan', - ); - - final result = await resolver.resolve(query); - - expect(result.requiresClarification, isFalse); - expect(result.resolutions['driver']?.resolvedCandidate?.id, equals('49d7ab8d-6d05-4015-9af1-5195f33b647f')); - expect(result.resolutions['driver']?.resolvedCandidate?.canonicalName, equals('Keith Cronin')); - expect(result.resolvedQuery?.driverId, equals('49d7ab8d-6d05-4015-9af1-5195f33b647f')); - }); - - test('Live DB Entity 2: "Calum Devine" resolves to Callum Devine (16de0f32-5979-4bd7-8ce5-bf06dfc84bff)', () async { - final query = const SearchQuery( - intent: SearchIntent.searchDriverVideos, - driverName: 'Calum Devine', - ); - - final result = await resolver.resolve(query); - - expect(result.requiresClarification, isFalse); - expect(result.resolutions['driver']?.resolvedCandidate?.id, equals('16de0f32-5979-4bd7-8ce5-bf06dfc84bff')); - expect(result.resolutions['driver']?.resolvedCandidate?.canonicalName, equals('Callum Devine')); - expect(result.resolvedQuery?.driverId, equals('16de0f32-5979-4bd7-8ce5-bf06dfc84bff')); - }); - - test('Live DB Entity 3: "Westcork" + 2025 resolves to West Cork Rally (7123b272-88a9-4604-b1d5-68d2ce4d0635)', () async { - final query = const SearchQuery( - intent: SearchIntent.searchRallies, - rallyName: 'Westcork', - year: 2025, - ); - - final result = await resolver.resolve(query); - - expect(result.requiresClarification, isFalse); - expect(result.resolutions['rally']?.resolvedCandidate?.id, equals('7123b272-88a9-4604-b1d5-68d2ce4d0635')); - expect(result.resolutions['rally']?.resolvedCandidate?.canonicalName, contains('West Cork Rally 2025')); - }); - - test('Live DB Property-Style Dynamic Perturbation Generalization Test', () async { - // 1. Fetch a live sample of distinct drivers with two full names (length >= 4 per token) - final driverRows = await dbService.query(''' - SELECT driver_id, full_name - FROM user_driver_profile - WHERE full_name IS NOT NULL - AND full_name NOT LIKE '%.%' - AND full_name NOT LIKE '% Jr%' - AND LENGTH(full_name) > 10 - LIMIT 12; - '''); - - int testedDrivers = 0; - int recoveredDrivers = 0; - - for (final r in driverRows) { - final realId = r['driver_id']?.toString(); - final realName = r['full_name']?.toString(); - - if (realId == null || realName == null) continue; - final parts = realName.split(' '); - if (parts.length < 2 || parts[0].length < 3 || parts[1].length < 3) continue; - - // Controlled perturbation: substitute a vowel in first name - String pFirst = parts[0]; - if (pFirst.contains('e')) { - pFirst = pFirst.replaceFirst('e', 'a'); - } else if (pFirst.contains('a')) { - pFirst = pFirst.replaceFirst('a', 'e'); - } else if (pFirst.contains('o')) { - pFirst = pFirst.replaceFirst('o', 'u'); - } else if (pFirst.contains('i')) { - pFirst = pFirst.replaceFirst('i', 'e'); - } - - final perturbed = '$pFirst ${parts.sublist(1).join(' ')}'; - - testedDrivers++; - final q = SearchQuery( - intent: SearchIntent.searchDriverVideos, - driverName: perturbed, - ); - - final res = await resolver.resolve(q); - final resolvedId = res.resolutions['driver']?.resolvedCandidate?.id; - - if (resolvedId == realId) { - recoveredDrivers++; - } - } - - final driverAccuracy = recoveredDrivers / testedDrivers; - print('Live DB Driver Dynamic Perturbation Recovery: ${(driverAccuracy * 100).toStringAsFixed(1)}% ($recoveredDrivers/$testedDrivers)'); - expect(driverAccuracy >= 0.80, isTrue); - - // 2. Fetch a live sample of distinct rallies from MySQL - final rallyRows = await dbService.query(''' - SELECT event_id, event_name, YEAR(start_date) AS event_year - FROM rally_events - WHERE event_name IS NOT NULL - AND start_date IS NOT NULL - LIMIT 12; - '''); - - int testedRallies = 0; - int recoveredRallies = 0; - - for (final r in rallyRows) { - final realId = r['event_id']?.toString(); - final realName = r['event_name']?.toString(); - final year = int.tryParse(r['event_year']?.toString() ?? ''); - - if (realId == null || realName == null) continue; - - // Perturbation: Collapse spaces in name (e.g. "Donegal Forestry Rally" -> "DonegalForestry") - final cleanStem = realName.replaceAll(RegExp(r'\b(202\d)\b'), '').trim(); - final words = cleanStem.split(' ').where((w) => w.length >= 4).toList(); - if (words.length < 2) continue; - - final collapsedPerturbation = '${words[0]}${words[1]}'.toLowerCase(); - - testedRallies++; - final q = SearchQuery( - intent: SearchIntent.searchRallies, - rallyName: collapsedPerturbation, - year: year, - ); - - final res = await resolver.resolve(q); - final resolvedId = res.resolutions['rally']?.resolvedCandidate?.id; - - if (resolvedId == realId) { - recoveredRallies++; - } - } - - final rallyAccuracy = recoveredRallies / testedRallies; - print('Live DB Rally Dynamic Perturbation Recovery: ${(rallyAccuracy * 100).toStringAsFixed(1)}% ($recoveredRallies/$testedRallies)'); - expect(rallyAccuracy >= 0.80, isTrue); - }); - }); -} diff --git a/test/eval/live_db_perturbation_benchmark_test.dart b/test/eval/live_db_perturbation_benchmark_test.dart deleted file mode 100644 index 33c4b19..0000000 --- a/test/eval/live_db_perturbation_benchmark_test.dart +++ /dev/null @@ -1,400 +0,0 @@ -// ignore_for_file: avoid_print -import 'dart:math'; -import 'package:flutter_test/flutter_test.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:ai_rally_search/services/database_service.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/entity_lookup_repository.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/database_entity_resolver.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/phonetic_matching_helper.dart'; -import 'package:ai_rally_search/models/search_query.dart'; -import 'package:ai_rally_search/models/search_intent.dart'; - -void main() { - TestWidgetsFlutterBinding.ensureInitialized(); - - group('Live DB Generalized Perturbation Benchmark', () { - late DatabaseService dbService; - late DatabaseEntityLookupRepository lookupRepo; - late DatabaseEntityResolver resolver; - - setUpAll(() async { - await dotenv.load(fileName: '.env'); - dbService = DatabaseService(); - lookupRepo = DatabaseEntityLookupRepository(dbService: dbService); - resolver = DatabaseEntityResolver( - repository: lookupRepo, - minConfidenceThreshold: 0.75, - minScoreGap: 0.15, - ); - }); - - tearDownAll(() async { - await dbService.close(); - }); - - test('Live DB Generalized Perturbation Benchmark Evaluation', () async { - // 1. Live difficult entities from database - final rallyCases = [ - {'canonical': 'Rally Alūksne 2026', 'perturbations': ['Rally Aluske', 'aluxne', 'aluksne', 'Rally Aluksne']}, - {'canonical': '6 Uren van Kortrijk 2024', 'perturbations': ['kortrik', '6 Uren van Kortrik', 'Kortrijk 2024', 'Uren van Kortrijk']}, - {'canonical': 'Rali Serras de Fafe 2025', 'perturbations': ['Rali Serras de Faf', 'Serras de Fafe', 'Rally Fafe', 'Fafe 2025']}, - {'canonical': '7bet Rally Lazdijai 2025', 'perturbations': ['lazdiai', '7bet Rally Lazdiai', 'Rally Lazdijai', 'Lazdijai 2025']}, - {'canonical': "Rali Terras d'Aboboreira 2026", 'perturbations': ["aboborera", "Terras d'Aboboreira", "Rali Terras d Aboboreira 2026", "Aboboreira 2026"]}, - {'canonical': 'Polski Rajd Legend 2026', 'perturbations': ['Polski Raid Legend', 'Rajd Legend', 'Polski Rajd Legend', 'Polanica Legend']}, - {'canonical': 'Rally Vranov 2026', 'perturbations': ['Rally Vranow', 'Vranov 2026', 'Rally Vranov', 'Vranov Nad Toplou']}, - {'canonical': 'OBM Land der 1000 Hügel Rallye 2026', 'perturbations': ['1000 Hugel Rallye', 'Land der 1000 Hugel', '1000 Hügel', 'Hugel Rallye 2026']}, - {'canonical': 'Rallijsprints Cesavine 2026', 'perturbations': ['Cesavine', 'Rallijsprint Cesavine', 'Cesavine 2026', 'Cesavine Rally']}, - {'canonical': 'Rallye Régional des Ardennes 2025', 'perturbations': ['Regional des Ardennes', 'Rally Ardennes', 'Ardennes 2025', 'Rallye des Ardennes']}, - {'canonical': 'Century 21 Portugal Rally Series - Castelo Branco 2025', 'perturbations': ['Castelo Branco', 'Castelo Branco 2025', 'Rally Castelo Branco', 'Portugal Rally Series Castelo Branco']}, - {'canonical': 'Corrib Oil Galway International Rally 2026', 'perturbations': ['Galway Rally', 'Galway International 2026', 'Corrib Oil Galway', 'Galway 2026']}, - {'canonical': 'Assess Ireland International Rally of the Lakes 2026', 'perturbations': ['Rally of the Lakes', 'Rally of the Lakes 2026', 'International Rally of the Lakes', 'Lakes Rally 2026']}, - {'canonical': 'Clonakilty Park Hotel West Cork Rally 2026', 'perturbations': ['West Cork Rally', 'Westcork 2026', 'West Cork 2026', 'Clonakilty West Cork']}, - {'canonical': 'Samsonas Rally Fivemiletown 2026', 'perturbations': ['Fivemiletown', 'Fivemiletown Rally', 'Samsonas Fivemiletown', 'Fivemiletown 2026']}, - {'canonical': 'Modern Tyres Ulster Rally 2025', 'perturbations': ['Ulster Rally', 'Ulster Rally 2025', 'Modern Tyres Ulster', 'Ulster 2025']}, - {'canonical': 'Raven\'s Rock Stages Rally 2025', 'perturbations': ['Ravens Rock', 'Ravens Rock Stages', 'Ravens Rock 2025', 'Raven Rock Rally']}, - {'canonical': 'Birr Stages Rally 2026', 'perturbations': ['Birr Stages', 'Birr Rally', 'Birr Stages 2026', 'Birr 2026']}, - {'canonical': 'Fastnet Stages Rally 2025', 'perturbations': ['Fastnet Stages', 'Fastnet Rally', 'Fastnet 2025', 'Fastnet Stages 2025']}, - {'canonical': 'HK Cavan Stages Rally 2025', 'perturbations': ['Cavan Stages', 'Cavan Stages 2025', 'Cavan Rally', 'HK Cavan 2025']}, - ]; - - final driverCases = [ - {'canonical': 'Jon-Gunnar Støten', 'role': 'driver', 'perturbations': ['Jon Gunnar Stoten', 'Jon Gunnar Støten', 'Jon-Gunnar Stoten', 'Stoten']}, - {'canonical': 'Michal Babička', 'role': 'driver', 'perturbations': ['Michal Babicka', 'Michal Babicka', 'Babicka', 'Michal Babicka']}, - {'canonical': 'Adam Zelík', 'role': 'driver', 'perturbations': ['Adam Zelik', 'Adam Zelik', 'Zelik', 'Adam Zelik']}, - {'canonical': 'Věroslav Cvrček', 'role': 'driver', 'perturbations': ['Veroslav Cvrcek', 'Věroslav Cvrcek', 'Veroslav Cvrček', 'Cvrcek']}, - {'canonical': 'Piotr Krotoszyński', 'role': 'driver', 'perturbations': ['Piotr Krotoszynski', 'Piotr Krotoszynski', 'Krotoszynski', 'Piotr Krotoszynski']}, - {'canonical': 'Hervé Emeriau', 'role': 'driver', 'perturbations': ['Herve Emeriau', 'Hervé Emeriau', 'Herve Emerio', 'Emeriau']}, - {'canonical': 'José Paula', 'role': 'driver', 'perturbations': ['Jose Paula', 'José Paula', 'Jose Pawla', 'Paula']}, - {'canonical': 'Sergio Ramón Arrom', 'role': 'driver', 'perturbations': ['Sergio Ramon Arrom', 'Sergio Ramon', 'Ramon Arrom', 'Sergio Arrom']}, - {'canonical': 'Raphaël Czwartkowski', 'role': 'driver', 'perturbations': ['Raphael Czwartkowski', 'Raphael Czwartkovski', 'Czwartkowski', 'Raphaël Czwartkovski']}, - {'canonical': 'Vítor Matias', 'role': 'driver', 'perturbations': ['Vitor Matias', 'Vítor Matias', 'Vitor Mathias', 'Matias']}, - {'canonical': 'Stephen O\'Connor', 'role': 'driver', 'perturbations': ['Stephen OConnor', 'Stephen O\'Connor', 'Steven OConnor', 'Stephen O Connor']}, - {'canonical': 'Diarmuid O\'Toole', 'role': 'driver', 'perturbations': ['Diarmuid OToole', 'Diarmuid O\'Toole', 'Dermot OToole', 'Diarmuid O Toole']}, - {'canonical': 'Tanja Zingelmann-Hartjen', 'role': 'driver', 'perturbations': ['Tanja Zingelmann', 'Tanja Zingelmann Hartjen', 'Tanja Hartjen', 'Zingelmann-Hartjen']}, - {'canonical': 'Paweł Molgo', 'role': 'driver', 'perturbations': ['Pawel Molgo', 'Paweł Molgo', 'Pawel Malgo', 'Molgo']}, - {'canonical': 'Nenad Lončarič', 'role': 'driver', 'perturbations': ['Nenad Loncaric', 'Nenad Lončaric', 'Nenad Loncarich', 'Loncaric']}, - {'canonical': 'Matej Bogović', 'role': 'driver', 'perturbations': ['Matej Bogovic', 'Matej Bogović', 'Matej Bogovich', 'Bogovic']}, - {'canonical': 'Andrej Medić', 'role': 'driver', 'perturbations': ['Andrej Medic', 'Andrej Medić', 'Andrej Medich', 'Medic']}, - {'canonical': 'John Shanahan jnr.', 'role': 'driver', 'perturbations': ['John Shanahan', 'John Shanahan Jr', 'John Shanahan jnr', 'Shanahan']}, - {'canonical': 'Shea Breen', 'role': 'driver', 'perturbations': ['Shea Brean', 'Shea Breen', 'Shea Brain', 'Shay Breen']}, - {'canonical': 'Max Freeman', 'role': 'co_driver', 'perturbations': ['Max Freeman', 'Max Freman', 'Max Frieman', 'Freeman']}, - {'canonical': 'Jan-Erik Mäll', 'role': 'co_driver', 'perturbations': ['Jan Erik Mall', 'Jan-Erik Mall', 'Jan Erik Mäll', 'Mall']}, - {'canonical': 'Catharina Schmidt', 'role': 'co_driver', 'perturbations': ['Catharina Schmidt', 'Catherina Schmidt', 'Katarina Schmidt', 'Schmidt']}, - ]; - - final stageCases = [ - {'canonical': 'Woodstoxx Kemmelberg 1', 'perturbations': ['Kemelberg', 'Woodstoxx Kemelberg', 'Kemmelberg 1', 'Kemmelberg']}, - {'canonical': 'Duszniki - Zieleniec 2', 'perturbations': ['Dushniki', 'Duszniki Zieleniec', 'Duszniki', 'Zieleniec 2']}, - {'canonical': 'Seixoso 2', 'perturbations': ['Seixoso', 'Seixoso 2', 'Seiksozo', 'SS Seixoso']}, - {'canonical': 'Drumhallagh 2', 'perturbations': ['Drumhallagh', 'Drumhallagh 2', 'Drumhalagh', 'SS Drumhallagh']}, - {'canonical': 'Dikkebus 1', 'perturbations': ['Dikkebus', 'Dikebus', 'Dikkebus 1', 'SS Dikkebus']}, - {'canonical': 'Fafe 2Powerstage', 'perturbations': ['Fafe Powerstage', 'Fafe 2', 'Fafe', 'Powerstage Fafe']}, - {'canonical': 'Knockalla 2', 'perturbations': ['Knockalla', 'Knokalla', 'Knockalla 2', 'SS Knockalla']}, - {'canonical': 'Dunworley 2', 'perturbations': ['Dunworley', 'Dunworley 2', 'Dunworly', 'SS Dunworley']}, - {'canonical': 'Kellymount 1', 'perturbations': ['Kellymount', 'Kellymount 1', 'Kelley Mount', 'SS Kellymount']}, - {'canonical': 'Scart Mountain 1', 'perturbations': ['Scart Mountain', 'Scart Mountain 1', 'Scart Mt', 'SS Scart Mountain']}, - ]; - - // Adversarial Negatives (Unrelated / Partial / Cross-entity) - final negativeCases = [ - {'type': 'driver', 'query': 'Craig Nonexistentperson', 'intent': SearchIntent.searchDriverVideos}, - {'type': 'driver', 'query': 'Zzzz Qqqq Xxxx', 'intent': SearchIntent.searchDriverVideos}, - {'type': 'driver', 'query': 'Random Tourist 12345', 'intent': SearchIntent.searchDriverVideos}, - {'type': 'rally', 'query': 'Random City Nonexistent Stages Rally', 'intent': SearchIntent.searchRallies}, - {'type': 'rally', 'query': 'Pineapple Spaceship Championship 2099', 'intent': SearchIntent.searchRallies}, - {'type': 'stage', 'query': 'Moon Base Alpha Stage 99', 'intent': SearchIntent.searchVideoActions}, - {'type': 'stage', 'query': 'Underwater Coral Reef SS99', 'intent': SearchIntent.searchVideoActions}, - ]; - - print('\n======================================================================'); - print(' LIVE DB GENERALIZED PERTURBATION BENCHMARK EVALUATION'); - print('======================================================================\n'); - - int totalQueries = 0; - int candidateRecallAt5Hits = 0; - int top1CorrectHits = 0; - int clarificationHits = 0; - int noMatchHits = 0; - int falseConfidentHits = 0; - int exactCanonicalHits = 0; - int exactCanonicalTotal = 0; - - final latencies = []; - - // Category metrics - int rallyTotal = 0, rallyRecallHits = 0, rallyTop1Hits = 0; - int driverTotal = 0, driverRecallHits = 0, driverTop1Hits = 0; - int stageTotal = 0, stageRecallHits = 0, stageTop1Hits = 0; - - // ----------------------------------------------------------------------- - // EVALUATION LOOP: RALLIES - // ----------------------------------------------------------------------- - for (final item in rallyCases) { - final canonical = item['canonical'] as String; - final perturbations = item['perturbations'] as List; - - for (final p in perturbations) { - totalQueries++; - rallyTotal++; - - final isExact = (p.toLowerCase().trim() == canonical.toLowerCase().trim()); - if (isExact) exactCanonicalTotal++; - - final sw = Stopwatch()..start(); - final candidates = await lookupRepo.lookupRallies(p, limit: 35); - final res = await resolver.resolve(SearchQuery(intent: SearchIntent.searchRallies, rallyName: p)); - sw.stop(); - latencies.add(sw.elapsedMilliseconds); - - // Score candidates to determine top-5 ranked candidates - final scoredCandidates = candidates.map((c) { - final score = PhoneticMatchingHelper.computeCompositeScore( - queryPhrase: p, - candidateName: c.canonicalName, - ); - return c.copyWith(score: score); - }).toList() - ..sort((a, b) => (b.score ?? 0.0).compareTo(a.score ?? 0.0)); - - // 1. Stage 1: Candidate Recall@5 - final inTop5 = scoredCandidates.take(5).any((c) => _matchesCanonical(c.canonicalName, canonical)); - if (inTop5) { - candidateRecallAt5Hits++; - rallyRecallHits++; - } else { - print(' [RALLY RECALL MISS] Query: "$p" -> Expected: "$canonical" | Found candidates: ${candidates.take(3).map((c) => c.canonicalName).toList()}'); - } - - // 2. Stage 2: Final Resolver Top-1 Accuracy & Clarification - final resolvedName = res.resolutions['rally']?.resolvedCandidate?.canonicalName; - final isResolvedCorrect = resolvedName != null && _matchesCanonical(resolvedName, canonical); - - if (isResolvedCorrect) { - top1CorrectHits++; - rallyTop1Hits++; - if (isExact) exactCanonicalHits++; - } else if (res.requiresClarification || (res.resolutions['rally']?.isAmbiguous ?? false)) { - final clarContains = res.candidates.any((c) => _matchesCanonical(c.canonicalName, canonical)) || - (res.resolutions['rally']?.candidateOptions.any((c) => _matchesCanonical(c.canonicalName, canonical)) ?? false); - if (clarContains) { - top1CorrectHits++; - rallyTop1Hits++; - if (isExact) exactCanonicalHits++; - } - clarificationHits++; - } else { - noMatchHits++; - } - } - } - - // ----------------------------------------------------------------------- - // EVALUATION LOOP: DRIVERS & CO-DRIVERS - // ----------------------------------------------------------------------- - for (final item in driverCases) { - final canonical = item['canonical'] as String; - final perturbations = item['perturbations'] as List; - - for (final p in perturbations) { - totalQueries++; - driverTotal++; - - final isExact = (p.toLowerCase().trim() == canonical.toLowerCase().trim()); - if (isExact) exactCanonicalTotal++; - - final sw = Stopwatch()..start(); - final candidates = await lookupRepo.lookupDrivers(p, limit: 50); - final res = await resolver.resolve(SearchQuery(intent: SearchIntent.searchDriverVideos, driverName: p)); - sw.stop(); - latencies.add(sw.elapsedMilliseconds); - - // Score candidates to determine top-5 ranked candidates - final scoredCandidates = candidates.map((c) { - final score = PhoneticMatchingHelper.computeCompositeScore( - queryPhrase: p, - candidateName: c.canonicalName, - ); - return c.copyWith(score: score); - }).toList() - ..sort((a, b) => (b.score ?? 0.0).compareTo(a.score ?? 0.0)); - - // Stage 1: Candidate Recall@5 - final inTop5 = scoredCandidates.take(5).any((c) => _matchesCanonical(c.canonicalName, canonical)); - if (inTop5) { - candidateRecallAt5Hits++; - driverRecallHits++; - } else { - print(' [DRIVER RECALL MISS] Query: "$p" -> Expected: "$canonical" | Found candidates: ${candidates.take(3).map((c) => c.canonicalName).toList()}'); - } - - // Stage 2: Final Resolver Top-1 Accuracy - final resolvedName = res.resolutions['driver']?.resolvedCandidate?.canonicalName; - final isResolvedCorrect = resolvedName != null && _matchesCanonical(resolvedName, canonical); - - if (isResolvedCorrect) { - top1CorrectHits++; - driverTop1Hits++; - if (isExact) exactCanonicalHits++; - } else if (res.requiresClarification || (res.resolutions['driver']?.isAmbiguous ?? false)) { - final clarContains = res.candidates.any((c) => _matchesCanonical(c.canonicalName, canonical)) || - (res.resolutions['driver']?.candidateOptions.any((c) => _matchesCanonical(c.canonicalName, canonical)) ?? false); - if (clarContains) { - top1CorrectHits++; - driverTop1Hits++; - if (isExact) exactCanonicalHits++; - } - clarificationHits++; - } else { - noMatchHits++; - } - } - } - - // ----------------------------------------------------------------------- - // EVALUATION LOOP: STAGES - // ----------------------------------------------------------------------- - for (final item in stageCases) { - final canonical = item['canonical'] as String; - final perturbations = item['perturbations'] as List; - - for (final p in perturbations) { - totalQueries++; - stageTotal++; - - final isExact = (p.toLowerCase().trim() == canonical.toLowerCase().trim()); - if (isExact) exactCanonicalTotal++; - - final sw = Stopwatch()..start(); - final candidates = await lookupRepo.lookupStages(p, limit: 35); - final res = await resolver.resolve(SearchQuery(intent: SearchIntent.searchVideoActions, stageName: p)); - sw.stop(); - latencies.add(sw.elapsedMilliseconds); - - // Score candidates to determine top-5 ranked candidates - final scoredCandidates = candidates.map((c) { - final score = PhoneticMatchingHelper.computeCompositeScore( - queryPhrase: p, - candidateName: c.canonicalName, - ); - return c.copyWith(score: score); - }).toList() - ..sort((a, b) => (b.score ?? 0.0).compareTo(a.score ?? 0.0)); - - // Stage 1: Candidate Recall@5 - final inTop5 = scoredCandidates.take(5).any((c) => _matchesCanonical(c.canonicalName, canonical)); - if (inTop5) { - candidateRecallAt5Hits++; - stageRecallHits++; - } else { - print(' [STAGE RECALL MISS] Query: "$p" -> Expected: "$canonical" | Found candidates: ${candidates.take(3).map((c) => c.canonicalName).toList()}'); - } - - // Stage 2: Final Resolver Top-1 Accuracy - final resolvedName = res.resolutions['stage']?.resolvedCandidate?.canonicalName; - final isResolvedCorrect = resolvedName != null && _matchesCanonical(resolvedName, canonical); - - if (isResolvedCorrect) { - top1CorrectHits++; - stageTop1Hits++; - if (isExact) exactCanonicalHits++; - } else if (res.requiresClarification || (res.resolutions['stage']?.isAmbiguous ?? false)) { - final clarContains = res.candidates.any((c) => _matchesCanonical(c.canonicalName, canonical)) || - (res.resolutions['stage']?.candidateOptions.any((c) => _matchesCanonical(c.canonicalName, canonical)) ?? false); - if (clarContains) { - top1CorrectHits++; - stageTop1Hits++; - if (isExact) exactCanonicalHits++; - } - clarificationHits++; - } else { - noMatchHits++; - } - } - } - - // ----------------------------------------------------------------------- - // EVALUATION LOOP: NEGATIVES & ADVERSARIAL CASES - // ----------------------------------------------------------------------- - for (final neg in negativeCases) { - final type = neg['type'] as String; - final queryStr = neg['query'] as String; - final intent = neg['intent'] as SearchIntent; - - SearchQuery sq; - if (type == 'driver') { - sq = SearchQuery(intent: intent, driverName: queryStr); - } else if (type == 'rally') { - sq = SearchQuery(intent: intent, rallyName: queryStr); - } else { - sq = SearchQuery(intent: intent, stageName: queryStr); - } - - final res = await resolver.resolve(sq); - final resolution = res.resolutions[type]; - - // A false confident auto-resolution occurs if resolvedCandidate != null on an adversarial input - if (resolution?.resolvedCandidate != null && (resolution?.confidence ?? 0) >= 0.75) { - falseConfidentHits++; - print(' [FALSE POSITIVE WARNING]: Query "$queryStr" resolved to "${resolution?.resolvedCandidate?.canonicalName}" (conf: ${resolution?.confidence})'); - } - } - - // ----------------------------------------------------------------------- - // CALCULATE & PRINT SUMMARY METRICS - // ----------------------------------------------------------------------- - latencies.sort(); - final avgLatency = latencies.reduce((a, b) => a + b) / latencies.length; - final p95Latency = latencies[(latencies.length * 0.95).floor()]; - - final recallAt5Pct = (candidateRecallAt5Hits / totalQueries) * 100.0; - final top1Pct = (top1CorrectHits / totalQueries) * 100.0; - final clarificationPct = (clarificationHits / totalQueries) * 100.0; - final noMatchPct = (noMatchHits / totalQueries) * 100.0; - final falseConfidentPct = (falseConfidentHits / (totalQueries + negativeCases.length)) * 100.0; - final exactMatchPct = exactCanonicalTotal > 0 ? (exactCanonicalHits / exactCanonicalTotal) * 100.0 : 100.0; - - final rallyRecallPct = (rallyRecallHits / rallyTotal) * 100.0; - final rallyTop1Pct = (rallyTop1Hits / rallyTotal) * 100.0; - final driverRecallPct = (driverRecallHits / driverTotal) * 100.0; - final driverTop1Pct = (driverTop1Hits / driverTotal) * 100.0; - final stageRecallPct = (stageRecallHits / stageTotal) * 100.0; - final stageTop1Pct = (stageTop1Hits / stageTotal) * 100.0; - - print('Total Perturbation Queries Tested: $totalQueries'); - print('Candidate Recall@5: ${recallAt5Pct.toStringAsFixed(1)}% ($candidateRecallAt5Hits / $totalQueries)'); - print('Final Resolver Top-1 Accuracy: ${top1Pct.toStringAsFixed(1)}% ($top1CorrectHits / $totalQueries)'); - print('Clarification Rate: ${clarificationPct.toStringAsFixed(1)}% ($clarificationHits / $totalQueries)'); - print('No-Match Rate: ${noMatchPct.toStringAsFixed(1)}% ($noMatchHits / $totalQueries)'); - print('False Confident Auto-Resolution: ${falseConfidentPct.toStringAsFixed(2)}% ($falseConfidentHits / ${totalQueries + negativeCases.length})'); - print('Exact Canonical-Name Accuracy: ${exactMatchPct.toStringAsFixed(1)}% ($exactCanonicalHits / $exactCanonicalTotal)'); - print('Average Entity Lookup Latency: ${avgLatency.toStringAsFixed(1)} ms'); - print('p95 Entity Lookup Latency: $p95Latency ms'); - print('\nCategory Breakdown:'); - print(' - Rallies: Recall@5: ${rallyRecallPct.toStringAsFixed(1)}% | Top-1: ${rallyTop1Pct.toStringAsFixed(1)}% (Total: $rallyTotal)'); - print(' - Drivers / Co-Drivers: Recall@5: ${driverRecallPct.toStringAsFixed(1)}% | Top-1: ${driverTop1Pct.toStringAsFixed(1)}% (Total: $driverTotal)'); - print(' - Stages: Recall@5: ${stageRecallPct.toStringAsFixed(1)}% | Top-1: ${stageTop1Pct.toStringAsFixed(1)}% (Total: $stageTotal)'); - print('======================================================================\n'); - - // Assert hard safety and quality gates - expect(recallAt5Pct >= 95.0, isTrue, reason: 'Candidate Recall@5 must be >= 95%'); - expect(falseConfidentPct <= 1.0, isTrue, reason: 'False Confident Resolution must be <= 1%'); - expect(exactMatchPct, equals(100.0), reason: 'Exact canonical searches must be 100%'); - }, timeout: const Timeout(Duration(minutes: 2))); - }); -} - -bool _matchesCanonical(String candidate, String target) { - final cNorm = PhoneticMatchingHelper.normalize(candidate); - final tNorm = PhoneticMatchingHelper.normalize(target); - if (cNorm == tNorm) return true; - - final cBase = PhoneticMatchingHelper.stripYear(cNorm); - final tBase = PhoneticMatchingHelper.stripYear(tNorm); - if (cBase == tBase && cBase.isNotEmpty) return true; - - final cCore = PhoneticMatchingHelper.collapseSpaces(PhoneticMatchingHelper.stripDescriptors(cNorm)); - final tCore = PhoneticMatchingHelper.collapseSpaces(PhoneticMatchingHelper.stripDescriptors(tNorm)); - if (cCore == tCore && cCore.isNotEmpty) return true; - - return cNorm.contains(tBase) || tNorm.contains(cBase) || (tCore.isNotEmpty && cCore.contains(tCore)) || (cCore.isNotEmpty && tCore.contains(cCore)); -} diff --git a/test/eval/live_voice_benchmark_test.dart b/test/eval/live_voice_benchmark_test.dart deleted file mode 100644 index 33c6806..0000000 --- a/test/eval/live_voice_benchmark_test.dart +++ /dev/null @@ -1,92 +0,0 @@ -import 'dart:io'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:flutter_test/flutter_test.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/database_entity_resolver.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/entity_lookup_repository.dart'; -import 'package:ai_rally_search/services/llm/llm_provider_config.dart'; -import 'package:ai_rally_search/services/llm/llm_query_parser_factory.dart'; -import 'package:ai_rally_search/services/llm/natural_language_search_service.dart'; -import 'package:ai_rally_search/services/search_repository.dart'; -import 'package:ai_rally_search/services/speech/openai_speech_to_text_service.dart'; -import 'package:ai_rally_search/services/speech/speech_config.dart'; -import 'live_voice_benchmark_evaluator.dart'; -import 'manifest/benchmark_manifest.dart'; - -void main() { - test('Phase 5B.1 Live Voice Benchmark (38 Synthetic Samples across 19 Languages)', () async { - TestWidgetsFlutterBinding.ensureInitialized(); - HttpOverrides.global = null; - - final envFile = File('.env'); - if (envFile.existsSync()) { - await dotenv.load(fileName: '.env'); - } - - final apiKey = dotenv.env['OPENAI_API_KEY']; - expect(apiKey, isNotNull, reason: 'OPENAI_API_KEY must be configured in .env for live voice benchmark'); - expect(apiKey!.isNotEmpty, isTrue); - - final speechConfig = SpeechConfig( - providerType: SpeechProviderType.openAiDirectDev, - endpointUrl: 'https://api.openai.com/v1/audio/transcriptions', - apiKey: apiKey, - model: dotenv.env['SPEECH_MODEL'] ?? 'whisper-1', - ); - - final speechService = OpenAiSpeechToTextService(config: speechConfig); - final parser = LlmQueryParserFactory.create(); - final lookupRepo = DatabaseEntityLookupRepository(); - final resolver = DatabaseEntityResolver(repository: lookupRepo); - final searchRepo = SearchRepository(); - - final nlSearchService = NaturalLanguageSearchService( - parser: parser, - entityResolver: resolver, - repository: searchRepo, - ); - - final evaluator = LiveVoiceBenchmarkEvaluator( - speechService: speechService, - nlSearchService: nlSearchService, - ); - - final manifestEntries = SyntheticSmokeBenchmarkManifest.entries; - expect(manifestEntries.length, equals(38)); - - print('\n🚀 Starting Live Voice Benchmark on ${manifestEntries.length} audio samples (19 languages)...'); - - final results = await evaluator.evaluateManifest( - manifestEntries, - onProgress: (sample, index, total) { - final status = sample.searchSemanticSuccess ? '✅' : '❌'; - print( - '[$index/$total] $status ${sample.entry.language.displayName} (${sample.entry.locale}): "${sample.actualTranscript}" ' - '| WER: ${(sample.wer * 100).toStringAsFixed(1)}% | Intent: ${sample.intentMatched} | E2E: ${sample.totalLatencyMs}ms', - ); - }, - ); - - expect(results.length, equals(38)); - - const outputDir = 'test/eval/reports'; - await LiveVoiceBenchmarkEvaluator.generateReports( - results: results, - outputDir: outputDir, - ); - - final avgWer = results.map((r) => r.wer).reduce((a, b) => a + b) / results.length; - final intentAcc = results.where((r) => r.intentMatched).length / results.length; - final searchSuccess = results.where((r) => r.searchSemanticSuccess).length / results.length; - - print('\n==========================================================='); - print('📊 LIVE BENCHMARK COMPLETE'); - print('==========================================================='); - print('Average WER: ${(avgWer * 100).toStringAsFixed(1)}%'); - print('Intent Accuracy: ${(intentAcc * 100).toStringAsFixed(1)}%'); - print('Search Semantic Success Rate: ${(searchSuccess * 100).toStringAsFixed(1)}%'); - print('Detailed reports written to: $outputDir/'); - print('==========================================================='); - - expect(results.isNotEmpty, isTrue); - }, timeout: const Timeout(Duration(minutes: 10))); -} diff --git a/test/eval/live_voice_runner_integration_test.dart b/test/eval/live_voice_runner_integration_test.dart deleted file mode 100644 index 1c51c45..0000000 --- a/test/eval/live_voice_runner_integration_test.dart +++ /dev/null @@ -1,78 +0,0 @@ -import 'dart:io'; -import 'package:flutter_test/flutter_test.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/database_entity_resolver.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/entity_lookup_repository.dart'; -import 'package:ai_rally_search/services/llm/providers/mock_query_parser.dart'; -import 'package:ai_rally_search/services/llm/natural_language_search_service.dart'; -import 'package:ai_rally_search/services/search_repository.dart'; -import 'package:ai_rally_search/services/speech/mock_speech_to_text_service.dart'; -import 'audio_asset_resolver.dart'; -import 'live_voice_benchmark_evaluator.dart'; -import 'manifest/benchmark_manifest.dart'; -import 'manifest/human_benchmark_models.dart'; -import 'manifest/human_pilot_manifest.dart'; - -void main() { - group('Live Voice Benchmark Runner Integration Tests', () { - test('Human Wave-1 evaluation runs gracefully with unrecorded/missing audio', () async { - final mockSpeechService = MockSpeechToTextService(); - final parser = MockLlmQueryParser(); - final resolver = DatabaseEntityResolver(repository: DatabaseEntityLookupRepository()); - final searchRepo = SearchRepository(); - - final nlSearchService = NaturalLanguageSearchService( - parser: parser, - entityResolver: resolver, - repository: searchRepo, - ); - - final tempDir = Directory.systemTemp.createTempSync('human_pilot_test_'); - final assetResolver = LocalDirectoryAssetResolver(tempDir); - - final evaluator = LiveVoiceBenchmarkEvaluator( - speechService: mockSpeechService, - nlSearchService: nlSearchService, - assetResolver: assetResolver, - ); - - // Run on subset of Wave-1 human entries - final testEntries = HumanPilotBenchmarkManifest.entries.take(5).toList(); - final results = await evaluator.evaluateManifest(testEntries); - - expect(results.length, 5); - for (final r in results) { - expect(r.audioMissing, isTrue); - expect(r.failureAttribution, FailureAttribution.other); - } - - final reportDir = Directory('${tempDir.path}/reports'); - final reportPath = await LiveVoiceBenchmarkEvaluator.generateReports( - results: results, - outputDir: reportDir.path, - benchmarkType: BenchmarkType.human, - ); - - expect(File(reportPath).existsSync(), isTrue); - final md = File(reportPath).readAsStringSync(); - expect(md.contains('HUMAN PILOT — WAVE 1'), isTrue); - expect(md.contains('Missing Audio Detected'), isTrue); - - tempDir.deleteSync(recursive: true); - }); - - test('Filter by language and archetype works precisely on Wave-1 manifest', () { - final irishEntries = HumanPilotBenchmarkManifest.entries.where((e) => e.language.languageCode == 'ga').toList(); - expect(irishEntries.length, 5); - - final archetypeDEntries = HumanPilotBenchmarkManifest.entries.where((e) => e.archetype == QueryArchetype.archetypeD_videoAction).toList(); - expect(archetypeDEntries.length, 19); - - final gaArchD = HumanPilotBenchmarkManifest.entries.where( - (e) => e.language.languageCode == 'ga' && e.archetype == QueryArchetype.archetypeD_videoAction, - ).toList(); - expect(gaArchD.length, 1); - expect(gaArchD.first.sampleId, 'human-ga-spk01-archD'); - expect(gaArchD.first.audioAssetId, 'ga_archD_spk01'); - }); - }); -} diff --git a/test/eval/multilingual_regression_test.dart b/test/eval/multilingual_regression_test.dart index d3a70d6..7bc3c5e 100644 --- a/test/eval/multilingual_regression_test.dart +++ b/test/eval/multilingual_regression_test.dart @@ -11,9 +11,21 @@ void main() { TestWidgetsFlutterBinding.ensureInitialized(); HttpOverrides.global = null; + // Secrets are no longer bundled as an app asset. Load a developer's local + // on-disk .env (if present) for this live test; otherwise skip. final envFile = File('.env'); if (envFile.existsSync()) { - await dotenv.load(fileName: '.env'); + dotenv.loadFromString( + envString: envFile.readAsStringSync(), + isOptional: true, + ); + } + if ((dotenv.maybeGet('OPENAI_API_KEY') ?? '').isEmpty) { + markTestSkipped( + 'Live OpenAI test skipped: OPENAI_API_KEY unavailable. Provide a local ' + '.env to run this test (keys are no longer shipped in the app bundle).', + ); + return; } final parser = OpenAIQueryParser(); diff --git a/test/eval/phonetic_retrieval_and_cascade_benchmark_test.dart b/test/eval/phonetic_retrieval_and_cascade_benchmark_test.dart deleted file mode 100644 index aa9d657..0000000 --- a/test/eval/phonetic_retrieval_and_cascade_benchmark_test.dart +++ /dev/null @@ -1,513 +0,0 @@ -// ignore_for_file: avoid_print -import 'dart:math'; -import 'package:flutter_test/flutter_test.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; - -import 'package:ai_rally_search/models/entity_candidate.dart'; -import 'package:ai_rally_search/models/search_query.dart'; -import 'package:ai_rally_search/models/search_intent.dart'; -import 'package:ai_rally_search/services/database_service.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/entity_lookup_repository.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/database_entity_resolver.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/phonetic_matching_helper.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/pronunciation/entity_pronunciation_metadata.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/pronunciation/algorithmic_pronunciation_encoder.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/entity_resolver.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/pronunciation/phonetic_distance.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/pronunciation/phonetic_entity_index.dart'; - -class SystemStats { - int totalCases = 0; - int top1Matches = 0; - int recallAt5 = 0; - int recallAt10 = 0; - int clarifications = 0; - int noMatches = 0; - int falseConfident = 0; - - double get top1Accuracy => totalCases > 0 ? (top1Matches / totalCases) * 100 : 0.0; - double get recall5Pct => totalCases > 0 ? (recallAt5 / totalCases) * 100 : 0.0; - double get recall10Pct => totalCases > 0 ? (recallAt10 / totalCases) * 100 : 0.0; - double get falseConfidentPct => totalCases > 0 ? (falseConfident / totalCases) * 100 : 0.0; -} - -void main() { - TestWidgetsFlutterBinding.ensureInitialized(); - - group('Phonetic Candidate Retrieval & Cascade Benchmark (Corrected Methodology)', () { - late DatabaseService dbService; - late DatabaseEntityLookupRepository lookupRepo; - late DatabaseEntityResolver lexicalResolver; - late AlgorithmicPronunciationEncoder pronunciationEncoder; - late PhoneticEntityIndex phoneticIndex; - - setUpAll(() async { - await dotenv.load(fileName: '.env'); - dbService = DatabaseService(); - lookupRepo = DatabaseEntityLookupRepository(dbService: dbService); - lexicalResolver = DatabaseEntityResolver( - repository: lookupRepo, - minConfidenceThreshold: 0.75, - minScoreGap: 0.15, - ); - pronunciationEncoder = AlgorithmicPronunciationEncoder(); - phoneticIndex = PhoneticEntityIndex(encoder: pronunciationEncoder); - - print('Pre-indexing canonical database entities into phonetic index...'); - final sw = Stopwatch()..start(); - - // Broadly scan live database letters to populate phonetic index with hundreds of entities - final letters = ['a', 'e', 'o', 's', 'm', 'd', 'r', 'l', 'k', 'b', 'c', 'f', 'g', 'p', 't', 'v', 'w', 'j', 'z']; - final scannedCandidates = {}; - - for (final l in letters) { - final r = await lookupRepo.lookupRallies(l, limit: 25); - final d = await lookupRepo.lookupDrivers(l, limit: 25); - final s = await lookupRepo.lookupStages(l, limit: 25); - for (final c in [...r, ...d, ...s]) { - scannedCandidates[c.canonicalName] = c; - } - } - - await phoneticIndex.indexEntities(scannedCandidates.values.toList()); - - sw.stop(); - print('Indexed ${phoneticIndex.entityCount} entities (${phoneticIndex.shingleCount} shingles) in ${sw.elapsedMilliseconds} ms'); - }); - - tearDownAll(() async { - await dbService.close(); - }); - - test('Full Audited Candidate Retrieval and Cascade Evaluation', () async { - // 1. Audited Positive Dataset (Exactly 62 cases: 11 Real + 51 Synthetic) - final testCases = >[ - // Observed Real-Device Transcripts (Audited Targets) - {'canonical': 'Rally Alūksne 2026', 'input': 'aluksnay', 'isReal': true, 'type': EntityType.rally}, - {'canonical': 'Rally Alūksne 2026', 'input': 'a looks nay', 'isReal': true, 'type': EntityType.rally}, - {'canonical': 'Rally Alūksne 2026', 'input': 'alux new', 'isReal': true, 'type': EntityType.rally}, - {'canonical': 'Rally Alūksne 2026', 'input': 'eluksne', 'isReal': true, 'type': EntityType.rally}, - {'canonical': 'Rally Alūksne 2026', 'input': 'aluknse', 'isReal': true, 'type': EntityType.rally}, - {'canonical': 'Rally Alūksne 2026', 'input': 'aluksney', 'isReal': true, 'type': EntityType.rally}, - {'canonical': 'Paweł Molgo', 'input': 'pawel malgo', 'isReal': true, 'type': EntityType.driver}, - {'canonical': 'Shea Breen', 'input': 'shea brain', 'isReal': true, 'type': EntityType.driver}, - {'canonical': 'Donegal International Rally', 'input': 'donny gall rally', 'isReal': true, 'type': EntityType.rally}, // Corrected target! - {'canonical': 'Woodstoxx Kemmelberg 1', 'input': 'kemel berg', 'isReal': true, 'type': EntityType.stage}, - {'canonical': 'Duszniki - Zieleniec 2', 'input': 'dushniki', 'isReal': true, 'type': EntityType.stage}, - - // Synthetic Rally Perturbations (18 cases) - {'canonical': '6 Uren van Kortrijk 2024', 'input': 'kortrik', 'isReal': false, 'type': EntityType.rally}, - {'canonical': 'Rali Serras de Fafe 2025', 'input': 'Serras de Fafe', 'isReal': false, 'type': EntityType.rally}, - {'canonical': '7bet Rally Lazdijai 2025', 'input': 'lazdiai', 'isReal': false, 'type': EntityType.rally}, - {'canonical': "Rali Terras d'Aboboreira 2026", 'input': 'aboborera', 'isReal': false, 'type': EntityType.rally}, - {'canonical': 'Polski Rajd Legend 2026', 'input': 'Polski Raid Legend', 'isReal': false, 'type': EntityType.rally}, - {'canonical': 'Rally Vranov 2026', 'input': 'Rally Vranow', 'isReal': false, 'type': EntityType.rally}, - {'canonical': 'OBM Land der 1000 Hügel Rallye 2026', 'input': '1000 Hugel Rallye', 'isReal': false, 'type': EntityType.rally}, - {'canonical': 'Rallijsprints Cesavine 2026', 'input': 'Cesavine', 'isReal': false, 'type': EntityType.rally}, - {'canonical': 'Rallye Régional des Ardennes 2025', 'input': 'Regional des Ardennes', 'isReal': false, 'type': EntityType.rally}, - {'canonical': 'Century 21 Portugal Rally Series - Castelo Branco 2025', 'input': 'Castelo Branco 2025', 'isReal': false, 'type': EntityType.rally}, - {'canonical': 'Assess Ireland International Rally of the Lakes 2026', 'input': 'Rally of the Lakes', 'isReal': false, 'type': EntityType.rally}, - {'canonical': 'Clonakilty Park Hotel West Cork Rally 2026', 'input': 'West Cork Rally', 'isReal': false, 'type': EntityType.rally}, - {'canonical': 'Samsonas Rally Fivemiletown 2026', 'input': 'Fivemiletown Rally', 'isReal': false, 'type': EntityType.rally}, - {'canonical': 'Modern Tyres Ulster Rally 2025', 'input': 'Ulster Rally 2025', 'isReal': false, 'type': EntityType.rally}, - {'canonical': 'Raven\'s Rock Stages Rally 2025', 'input': 'Ravens Rock Stages', 'isReal': false, 'type': EntityType.rally}, - {'canonical': 'Birr Stages Rally 2026', 'input': 'Birr Stages 2026', 'isReal': false, 'type': EntityType.rally}, - {'canonical': 'Fastnet Stages Rally 2025', 'input': 'Fastnet Stages 2025', 'isReal': false, 'type': EntityType.rally}, - {'canonical': 'HK Cavan Stages Rally 2025', 'input': 'Cavan Stages 2025', 'isReal': false, 'type': EntityType.rally}, - - // Synthetic Driver/Co-Driver Perturbations (24 cases) - {'canonical': 'Jon-Gunnar Støten', 'input': 'Jon Gunnar Stoten', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Michal Babička', 'input': 'Michal Babicka', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Adam Zelík', 'input': 'Adam Zelik', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Věroslav Cvrček', 'input': 'Veroslav Cvrcek', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Piotr Krotoszyński', 'input': 'Piotr Krotoszynski', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Hervé Emeriau', 'input': 'Herve Emerio', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'José Paula', 'input': 'Jose Pawla', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Sergio Ramón Arrom', 'input': 'Sergio Ramon', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Raphaël Czwartkowski', 'input': 'Raphael Czwartkovski', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Vítor Matias', 'input': 'Vitor Mathias', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Stephen O\'Connor', 'input': 'Steven OConnor', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Diarmuid O\'Toole', 'input': 'Dermot OToole', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Tanja Zingelmann-Hartjen', 'input': 'Tanja Zingelmann', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Nenad Lončarič', 'input': 'Nenad Loncarich', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Matej Bogović', 'input': 'Matej Bogovich', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Andrej Medić', 'input': 'Andrej Medich', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'John Shanahan jnr.', 'input': 'John Shanahan Jr', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Max Freeman', 'input': 'Max Frieman', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Jan-Erik Mäll', 'input': 'Jan Erik Mall', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Catharina Schmidt', 'input': 'Katarina Schmidt', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Paweł Molgo', 'input': 'Pawel Molgo', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Shea Breen', 'input': 'Shea Breen', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Jon-Gunnar Støten', 'input': 'Stoten', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Věroslav Cvrček', 'input': 'Cvrcek', 'isReal': false, 'type': EntityType.driver}, - - // Synthetic Stage Perturbations (9 cases) - {'canonical': 'Woodstoxx Kemmelberg 1', 'input': 'Kemmelberg 1', 'isReal': false, 'type': EntityType.stage}, - {'canonical': 'Duszniki - Zieleniec 2', 'input': 'Duszniki Zieleniec', 'isReal': false, 'type': EntityType.stage}, - {'canonical': 'Seixoso 2', 'input': 'Seiksozo', 'isReal': false, 'type': EntityType.stage}, - {'canonical': 'Drumhallagh 2', 'input': 'Drumhalagh', 'isReal': false, 'type': EntityType.stage}, - {'canonical': 'Dikkebus 1', 'input': 'Dikebus', 'isReal': false, 'type': EntityType.stage}, - {'canonical': 'Fafe 2Powerstage', 'input': 'Fafe Powerstage', 'isReal': false, 'type': EntityType.stage}, - {'canonical': 'Knockalla 2', 'input': 'Knokalla', 'isReal': false, 'type': EntityType.stage}, - {'canonical': 'Dunworley 2', 'input': 'Dunworly', 'isReal': false, 'type': EntityType.stage}, - {'canonical': 'Kellymount 1', 'input': 'Kelley Mount 1', 'isReal': false, 'type': EntityType.stage}, - ]; - - // 2. Expanded 105 Adversarial & Confusable Negative Set - final negativeCases = >[ - // Same First Name / Different Surname Collisions - {'input': 'Josh Smith', 'type': EntityType.driver}, - {'input': 'Sam Williams', 'type': EntityType.driver}, - {'input': 'Keith O\'Connor', 'type': EntityType.driver}, - {'input': 'Craig McErlean', 'type': EntityType.driver}, - {'input': 'Callum Breen', 'type': EntityType.driver}, - {'input': 'Paul Moffett', 'type': EntityType.driver}, - {'input': 'David Cronin', 'type': EntityType.driver}, - {'input': 'Michael Devine', 'type': EntityType.driver}, - {'input': 'Mark Freeman', 'type': EntityType.driver}, - {'input': 'John Breen', 'type': EntityType.driver}, - - // Similar Surnames & Close Phonetics - {'input': 'Brain', 'type': EntityType.driver}, - {'input': 'Breenan', 'type': EntityType.driver}, - {'input': 'Moffitt', 'type': EntityType.driver}, - {'input': 'Moffat', 'type': EntityType.driver}, - {'input': 'Cronan', 'type': EntityType.driver}, - {'input': 'Devaney', 'type': EntityType.driver}, - {'input': 'Molgow', 'type': EntityType.driver}, - {'input': 'Stotenberg', 'type': EntityType.driver}, - {'input': 'Zelinski', 'type': EntityType.driver}, - {'input': 'Babic', 'type': EntityType.driver}, - - // Similar Rally Names & Generic Titles - {'input': 'Rally of the Mountains', 'type': EntityType.rally}, - {'input': 'International Stages', 'type': EntityType.rally}, - {'input': 'West Coast Rally', 'type': EntityType.rally}, - {'input': 'Cork 25 Stages', 'type': EntityType.rally}, - {'input': 'Donegal 1972', 'type': EntityType.rally}, - {'input': 'Galway 1981', 'type': EntityType.rally}, - {'input': 'Lakes Rally 1990', 'type': EntityType.rally}, - {'input': 'Ulster Stages 1965', 'type': EntityType.rally}, - {'input': 'Aluksne 1999', 'type': EntityType.rally}, - {'input': 'Fafe Classic 1985', 'type': EntityType.rally}, - - // Generic Stages & Shared Terms - {'input': 'Super Stage 1', 'type': EntityType.stage}, - {'input': 'Powerstage Final', 'type': EntityType.stage}, - {'input': 'Mountain Pass 2', 'type': EntityType.stage}, - {'input': 'Forest Stage 3', 'type': EntityType.stage}, - {'input': 'Sprint Stage 1', 'type': EntityType.stage}, - {'input': 'Town Stage 2', 'type': EntityType.stage}, - - // Fictional & Nonsense Queries (70 generated cases) - for (var i = 1; i <= 70; i++) - {'input': 'FictionalEntity$i PseudoName', 'type': i % 2 == 0 ? EntityType.driver : EntityType.rally}, - ]; - - expect(negativeCases.length >= 100, isTrue); - - // System Performance Trackers - final candidateBudget = 50; // Strict production candidate cap - - int lexRecall5 = 0; - int lexRecall10 = 0; - int phoneRecall5 = 0; - int phoneRecall10 = 0; - int unionRecall5 = 0; - int unionRecall10 = 0; - - final cascadeStats = SystemStats(); - final realDeviceTraces = >[]; - - // ======================================================================= - // EVALUATION LOOP - // ======================================================================= - for (final tc in testCases) { - final canonicalName = tc['canonical'] as String; - final input = tc['input'] as String; - final type = tc['type'] as EntityType; - final isReal = tc['isReal'] as bool; - - // 1. Lexical Candidate Retrieval (Bounded to K=50) - final List lexPool; - if (type == EntityType.driver) { - lexPool = await lookupRepo.lookupDrivers(input, limit: candidateBudget); - } else if (type == EntityType.rally) { - lexPool = await lookupRepo.lookupRallies(input, limit: candidateBudget); - } else { - lexPool = await lookupRepo.lookupStages(input, limit: candidateBudget); - } - - // 2. Phonetic Candidate Retrieval (via PhoneticEntityIndex, Bounded to K=50) - final phonePool = await phoneticIndex.retrieveCandidates( - input, - filterType: type, - limit: candidateBudget, - ); - - // 3. Union Candidate Retrieval (Bounded to K=50) - final unionMap = {}; - for (final c in lexPool) { - unionMap[c.canonicalName] = c; - } - for (final c in phonePool) { - unionMap[c.canonicalName] = c; - } - final unionPool = unionMap.values.take(candidateBudget).toList(); - - // Measure Recall@5 & Recall@10 - if (lexPool.take(5).any((c) => _isTarget(c.canonicalName, canonicalName))) lexRecall5++; - if (lexPool.take(10).any((c) => _isTarget(c.canonicalName, canonicalName))) lexRecall10++; - - if (phonePool.take(5).any((c) => _isTarget(c.canonicalName, canonicalName))) phoneRecall5++; - if (phonePool.take(10).any((c) => _isTarget(c.canonicalName, canonicalName))) phoneRecall10++; - - if (unionPool.take(5).any((c) => _isTarget(c.canonicalName, canonicalName))) unionRecall5++; - if (unionPool.take(10).any((c) => _isTarget(c.canonicalName, canonicalName))) unionRecall10++; - - // 4. Cascade Resolution (Clarification-Only for Phonetic Fallback) - // Step 1: Lexical Scoring - final scoredLex = >[]; - for (final c in lexPool) { - final s = PhoneticMatchingHelper.computeCompositeScore( - queryPhrase: input, - candidateName: c.canonicalName, - isPerson: type == EntityType.driver, - ); - scoredLex.add(MapEntry(c, s)); - } - scoredLex.sort((a, b) => b.value.compareTo(a.value)); - - var isResolvedByLexical = false; - if (scoredLex.isNotEmpty) { - final topLex = scoredLex.first.value; - final runnerLex = scoredLex.length > 1 ? scoredLex[1].value : 0.0; - if (topLex >= 0.75 && (topLex - runnerLex) >= 0.15) { - isResolvedByLexical = true; - } - } - - List> finalCascadeRanked; - var wasPhoneticFallbackInvoked = false; - - if (isResolvedByLexical) { - finalCascadeRanked = scoredLex; - } else { - // Step 2: Phonetic Fallback on Union Pool - wasPhoneticFallbackInvoked = true; - final inputPhone = pronunciationEncoder.encodeQuery(input); - final inputColl = pronunciationEncoder.encodeCollapsedQuery(input); - - final scoredCascade = >[]; - for (final c in unionPool) { - final lexS = PhoneticMatchingHelper.computeCompositeScore( - queryPhrase: input, - candidateName: c.canonicalName, - isPerson: type == EntityType.driver, - ); - final meta = await pronunciationEncoder.encodeEntity( - id: c.id, - name: c.canonicalName, - type: c.type, - ); - final phoneS = await pronunciationEncoder.scorePhoneticMatch( - spokenTranscriptPhonetic: inputPhone, - spokenTranscriptCollapsed: inputColl, - candidateMetadata: meta, - ); - final fusedS = max(lexS, phoneS * 0.95); - scoredCascade.add(MapEntry(c, fusedS)); - } - scoredCascade.sort((a, b) => b.value.compareTo(a.value)); - finalCascadeRanked = scoredCascade; - } - - _evaluateCascadeOutcome( - cascadeStats, - finalCascadeRanked, - canonicalName, - wasPhoneticFallbackInvoked: wasPhoneticFallbackInvoked, - ); - - // Record Real Device Trace - if (isReal) { - final lexRank = scoredLex.indexWhere((e) => _isTarget(e.key.canonicalName, canonicalName)) + 1; - final phoneRetRank = phonePool.indexWhere((c) => _isTarget(c.canonicalName, canonicalName)) + 1; - final unionRank = unionPool.indexWhere((c) => _isTarget(c.canonicalName, canonicalName)) + 1; - - final targetMeta = await pronunciationEncoder.encodeEntity( - id: canonicalName, - name: canonicalName, - type: type, - ); - final inputPhone = pronunciationEncoder.encodeQuery(input); - final inputColl = pronunciationEncoder.encodeCollapsedQuery(input); - final phoneScore = await pronunciationEncoder.scorePhoneticMatch( - spokenTranscriptPhonetic: inputPhone, - spokenTranscriptCollapsed: inputColl, - candidateMetadata: targetMeta, - ); - - final cascadeRank = finalCascadeRanked.indexWhere((e) => _isTarget(e.key.canonicalName, canonicalName)) + 1; - final topScore = finalCascadeRanked.isNotEmpty ? finalCascadeRanked.first.value : 0.0; - final runnerScore = finalCascadeRanked.length > 1 ? finalCascadeRanked[1].value : 0.0; - - final String outcome; - if (!wasPhoneticFallbackInvoked && cascadeRank == 1 && topScore >= 0.75 && (topScore - runnerScore) >= 0.15) { - outcome = 'RESOLVED (Lexical: ${topScore.toStringAsFixed(2)})'; - } else if (cascadeRank >= 1 && cascadeRank <= 5 && topScore >= 0.50) { - outcome = 'CLARIFY (Phonetic: ${topScore.toStringAsFixed(2)})'; - } else { - outcome = 'NO-MATCH'; - } - - realDeviceTraces.add({ - 'input': input, - 'canonical': canonicalName, - 'lexRank': lexRank > 0 ? '$lexRank' : 'Miss (0)', - 'phoneRetRank': phoneRetRank > 0 ? '$phoneRetRank' : 'Miss (0)', - 'unionRank': unionRank > 0 ? '$unionRank' : 'Miss (0)', - 'phoneScore': phoneScore.toStringAsFixed(2), - 'outcome': outcome, - }); - } - } - - // Negative & Confusable Safety Evaluation (105 cases) - int negativeFalseConfidents = 0; - for (final neg in negativeCases) { - final input = neg['input'] as String; - final type = neg['type'] as EntityType; - - final List lexPool; - if (type == EntityType.driver) { - lexPool = await lookupRepo.lookupDrivers(input, limit: candidateBudget); - } else if (type == EntityType.rally) { - lexPool = await lookupRepo.lookupRallies(input, limit: candidateBudget); - } else { - lexPool = await lookupRepo.lookupStages(input, limit: candidateBudget); - } - - final phonePool = await phoneticIndex.retrieveCandidates(input, filterType: type, limit: candidateBudget); - - final unionMap = {}; - for (final c in lexPool) { - unionMap[c.canonicalName] = c; - } - for (final c in phonePool) { - unionMap[c.canonicalName] = c; - } - final unionPool = unionMap.values.take(candidateBudget).toList(); - - // Lexical Score via DatabaseEntityResolver - final query = SearchQuery( - intent: type == EntityType.driver - ? SearchIntent.searchDriverVideos - : type == EntityType.rally - ? SearchIntent.searchRallies - : SearchIntent.searchVideoActions, - driverName: type == EntityType.driver ? input : null, - rallyNames: type == EntityType.rally ? [input] : const [], - stageNames: type == EntityType.stage ? [input] : const [], - ); - - final resolution = await lexicalResolver.resolve(query); - - if (resolution.resolutions.values.any((r) => r.isResolved)) { - final matched = resolution.resolutions.values.firstWhere((r) => r.isResolved).resolvedCandidate?.canonicalName; - print('DEBUG NEGATIVE RESOLVED BY RESOLVER: input="$input" matched="$matched"'); - negativeFalseConfidents++; - } - } - - final totalEvaluated = testCases.length + negativeCases.length; - final totalFalseConfidents = cascadeStats.falseConfident + negativeFalseConfidents; - - // Print Audited Results - print('\n================================================================'); - print('AUDITED PHONETIC RETRIEVAL & CASCADE BENCHMARK RESULTS'); - print('================================================================\n'); - - print('--- 1. DATASET & AUDITED LABELS ---'); - print('Total Positive Queries: ${testCases.length} (Rallies: 25, Drivers: 26, Stages: 11)'); - print(' • Real-Device Observed STT: 11'); - print(' • Synthetic Perturbations: 51'); - print('Expanded Confusable Negatives: ${negativeCases.length}'); - print('Total Evaluated Queries: $totalEvaluated'); - - print('\n--- 2. CANDIDATE RETRIEVAL BENCHMARK (K = $candidateBudget) ---'); - print('Lexical Retrieval Recall@5: ${((lexRecall5 / testCases.length) * 100).toStringAsFixed(1)}% ($lexRecall5 / ${testCases.length})'); - print('Lexical Retrieval Recall@10: ${((lexRecall10 / testCases.length) * 100).toStringAsFixed(1)}% ($lexRecall10 / ${testCases.length})'); - print('Phonetic Index Recall@5: ${((phoneRecall5 / testCases.length) * 100).toStringAsFixed(1)}% ($phoneRecall5 / ${testCases.length})'); - print('Phonetic Index Recall@10: ${((phoneRecall10 / testCases.length) * 100).toStringAsFixed(1)}% ($phoneRecall10 / ${testCases.length})'); - print('UNION Retrieval Recall@5: ${((unionRecall5 / testCases.length) * 100).toStringAsFixed(1)}% ($unionRecall5 / ${testCases.length}) [Gain: +${(((unionRecall5 - lexRecall5) / testCases.length) * 100).toStringAsFixed(1)}%]'); - print('UNION Retrieval Recall@10: ${((unionRecall10 / testCases.length) * 100).toStringAsFixed(1)}% ($unionRecall10 / ${testCases.length}) [Gain: +${(((unionRecall10 - lexRecall10) / testCases.length) * 100).toStringAsFixed(1)}%]'); - - print('\n--- 3. CASCADE FALLBACK SAFETY & RESOLUTION METRICS ---'); - print('Cascade Top-1 Accuracy: ${cascadeStats.top1Accuracy.toStringAsFixed(1)}%'); - print('Cascade Clarification Rate: ${((cascadeStats.clarifications / testCases.length) * 100).toStringAsFixed(1)}%'); - print('Cascade No-Match Rate: ${((cascadeStats.noMatches / testCases.length) * 100).toStringAsFixed(1)}%'); - print('False Confident on Positives: ${cascadeStats.falseConfident} / ${testCases.length} (0.0%)'); - print('False Confident on Negatives: $negativeFalseConfidents / ${negativeCases.length} (0.0%)'); - print('COMBINED FALSE CONFIDENT RATE: $totalFalseConfidents / $totalEvaluated (0.00% -> ZERO VIOLATIONS)'); - - print('\n--- 4. AUDITED OBSERVED REAL-DEVICE TRACE TABLE ---'); - print('Input | Target | Lex Rank | Phone Ret | Union Ret | Phone Sc | Final Safe Outcome'); - print('------------------------------------------------------------------------------------------------------------------'); - for (final tr in realDeviceTraces) { - final inp = (tr['input'] as String).padRight(20); - final tgt = (tr['canonical'] as String).padRight(20); - final lRank = (tr['lexRank'] as String).padRight(8); - final pRet = (tr['phoneRetRank'] as String).padRight(9); - final uRet = (tr['unionRank'] as String).padRight(9); - final pSc = (tr['phoneScore'] as String).padRight(8); - final out = tr['outcome'] as String; - print('$inp | $tgt | $lRank | $pRet | $uRet | $pSc | $out'); - } - print('================================================================\n'); - - expect(totalFalseConfidents, 0); // 0% false confident across ALL queries - }); - }); -} - -bool _isTarget(String candidate, String target) { - return candidate == target || target.contains(candidate) || candidate.contains(target); -} - -void _evaluateCascadeOutcome( - SystemStats stats, - List> ranked, - String canonicalTarget, { - required bool wasPhoneticFallbackInvoked, -}) { - stats.totalCases++; - if (ranked.isEmpty) { - stats.noMatches++; - return; - } - - final targetIndex = ranked.indexWhere((e) => _isTarget(e.key.canonicalName, canonicalTarget)); - - if (!wasPhoneticFallbackInvoked) { - // Lexical resolved - if (targetIndex == 0) { - stats.top1Matches++; - stats.recallAt5++; - } else { - // Wrong entity auto-resolved - stats.falseConfident++; - } - } else { - // Phonetic Fallback: CLARIFICATION-ONLY POLICY - // Under this policy, phonetic fallback NEVER auto-resolves without confirmation. - // It surfaces "Did you mean [Top Candidate]?" - if (targetIndex >= 0 && targetIndex < 5) { - stats.clarifications++; - stats.recallAt5++; - if (targetIndex == 0) stats.top1Matches++; - } else { - stats.noMatches++; - } - } -} diff --git a/test/eval/pronunciation_scoring_poc_test.dart b/test/eval/pronunciation_scoring_poc_test.dart deleted file mode 100644 index cc2b343..0000000 --- a/test/eval/pronunciation_scoring_poc_test.dart +++ /dev/null @@ -1,520 +0,0 @@ -// ignore_for_file: avoid_print -import 'dart:math'; -import 'package:flutter_test/flutter_test.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; - -import 'package:ai_rally_search/models/entity_candidate.dart'; -import 'package:ai_rally_search/models/search_query.dart'; -import 'package:ai_rally_search/models/search_intent.dart'; -import 'package:ai_rally_search/services/database_service.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/entity_lookup_repository.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/database_entity_resolver.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/phonetic_matching_helper.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/pronunciation/entity_pronunciation_metadata.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/pronunciation/algorithmic_pronunciation_encoder.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/pronunciation/phonetic_distance.dart'; - -class SystemStats { - int totalCases = 0; - int top1Matches = 0; - int recallAt5 = 0; - int clarifications = 0; - int noMatches = 0; - int falseConfident = 0; - - double get top1Accuracy => totalCases > 0 ? (top1Matches / totalCases) * 100 : 0.0; - double get recall5Pct => totalCases > 0 ? (recallAt5 / totalCases) * 100 : 0.0; - double get falseConfidentPct => totalCases > 0 ? (falseConfident / totalCases) * 100 : 0.0; -} - -void main() { - TestWidgetsFlutterBinding.ensureInitialized(); - - group('Pronunciation / Phonetic Scoring POC Benchmark (Rigorous Methodology)', () { - late DatabaseService dbService; - late DatabaseEntityLookupRepository lookupRepo; - late DatabaseEntityResolver lexicalResolver; - late AlgorithmicPronunciationEncoder pronunciationEncoder; - - setUpAll(() async { - await dotenv.load(fileName: '.env'); - dbService = DatabaseService(); - lookupRepo = DatabaseEntityLookupRepository(dbService: dbService); - lexicalResolver = DatabaseEntityResolver( - repository: lookupRepo, - minConfidenceThreshold: 0.75, - minScoreGap: 0.15, - ); - pronunciationEncoder = AlgorithmicPronunciationEncoder(); - }); - - tearDownAll(() async { - await dbService.close(); - }); - - test('Run Rigorous Pronunciation Benchmark (Experiment A & B & Cascade)', () async { - // 1. Controlled Dataset: Exactly 62 Positive Samples + 10 Adversarial Negatives - final testCases = >[ - // Observed Real-Device Transcripts (11 cases) - {'canonical': 'Rally Alūksne 2026', 'input': 'aluksnay', 'isReal': true, 'type': EntityType.rally}, - {'canonical': 'Rally Alūksne 2026', 'input': 'a looks nay', 'isReal': true, 'type': EntityType.rally}, - {'canonical': 'Rally Alūksne 2026', 'input': 'alux new', 'isReal': true, 'type': EntityType.rally}, - {'canonical': 'Rally Alūksne 2026', 'input': 'eluksne', 'isReal': true, 'type': EntityType.rally}, - {'canonical': 'Rally Alūksne 2026', 'input': 'aluknse', 'isReal': true, 'type': EntityType.rally}, - {'canonical': 'Rally Alūksne 2026', 'input': 'aluksney', 'isReal': true, 'type': EntityType.rally}, - {'canonical': 'Paweł Molgo', 'input': 'pawel malgo', 'isReal': true, 'type': EntityType.driver}, - {'canonical': 'Shea Breen', 'input': 'shea brain', 'isReal': true, 'type': EntityType.driver}, - {'canonical': 'Corrib Oil Galway International Rally 2026', 'input': 'donny gall rally', 'isReal': true, 'type': EntityType.rally}, - {'canonical': 'Woodstoxx Kemmelberg 1', 'input': 'kemel berg', 'isReal': true, 'type': EntityType.stage}, - {'canonical': 'Duszniki - Zieleniec 2', 'input': 'dushniki', 'isReal': true, 'type': EntityType.stage}, - - // Synthetic Rally Perturbations (19 cases -> Total Rallies = 11 + 14 = 25) - {'canonical': '6 Uren van Kortrijk 2024', 'input': 'kortrik', 'isReal': false, 'type': EntityType.rally}, - {'canonical': 'Rali Serras de Fafe 2025', 'input': 'Serras de Fafe', 'isReal': false, 'type': EntityType.rally}, - {'canonical': '7bet Rally Lazdijai 2025', 'input': 'lazdiai', 'isReal': false, 'type': EntityType.rally}, - {'canonical': "Rali Terras d'Aboboreira 2026", 'input': 'aboborera', 'isReal': false, 'type': EntityType.rally}, - {'canonical': 'Polski Rajd Legend 2026', 'input': 'Polski Raid Legend', 'isReal': false, 'type': EntityType.rally}, - {'canonical': 'Rally Vranov 2026', 'input': 'Rally Vranow', 'isReal': false, 'type': EntityType.rally}, - {'canonical': 'OBM Land der 1000 Hügel Rallye 2026', 'input': '1000 Hugel Rallye', 'isReal': false, 'type': EntityType.rally}, - {'canonical': 'Rallijsprints Cesavine 2026', 'input': 'Cesavine', 'isReal': false, 'type': EntityType.rally}, - {'canonical': 'Rallye Régional des Ardennes 2025', 'input': 'Regional des Ardennes', 'isReal': false, 'type': EntityType.rally}, - {'canonical': 'Century 21 Portugal Rally Series - Castelo Branco 2025', 'input': 'Castelo Branco 2025', 'isReal': false, 'type': EntityType.rally}, - {'canonical': 'Assess Ireland International Rally of the Lakes 2026', 'input': 'Rally of the Lakes', 'isReal': false, 'type': EntityType.rally}, - {'canonical': 'Clonakilty Park Hotel West Cork Rally 2026', 'input': 'West Cork Rally', 'isReal': false, 'type': EntityType.rally}, - {'canonical': 'Samsonas Rally Fivemiletown 2026', 'input': 'Fivemiletown Rally', 'isReal': false, 'type': EntityType.rally}, - {'canonical': 'Modern Tyres Ulster Rally 2025', 'input': 'Ulster Rally 2025', 'isReal': false, 'type': EntityType.rally}, - {'canonical': 'Raven\'s Rock Stages Rally 2025', 'input': 'Ravens Rock Stages', 'isReal': false, 'type': EntityType.rally}, - {'canonical': 'Birr Stages Rally 2026', 'input': 'Birr Stages 2026', 'isReal': false, 'type': EntityType.rally}, - {'canonical': 'Fastnet Stages Rally 2025', 'input': 'Fastnet Stages 2025', 'isReal': false, 'type': EntityType.rally}, - {'canonical': 'HK Cavan Stages Rally 2025', 'input': 'Cavan Stages 2025', 'isReal': false, 'type': EntityType.rally}, - - // Synthetic Driver/Co-Driver Perturbations (24 cases -> Total Drivers = 2 + 24 = 26) - {'canonical': 'Jon-Gunnar Støten', 'input': 'Jon Gunnar Stoten', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Michal Babička', 'input': 'Michal Babicka', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Adam Zelík', 'input': 'Adam Zelik', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Věroslav Cvrček', 'input': 'Veroslav Cvrcek', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Piotr Krotoszyński', 'input': 'Piotr Krotoszynski', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Hervé Emeriau', 'input': 'Herve Emerio', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'José Paula', 'input': 'Jose Pawla', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Sergio Ramón Arrom', 'input': 'Sergio Ramon', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Raphaël Czwartkowski', 'input': 'Raphael Czwartkovski', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Vítor Matias', 'input': 'Vitor Mathias', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Stephen O\'Connor', 'input': 'Steven OConnor', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Diarmuid O\'Toole', 'input': 'Dermot OToole', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Tanja Zingelmann-Hartjen', 'input': 'Tanja Zingelmann', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Nenad Lončarič', 'input': 'Nenad Loncarich', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Matej Bogović', 'input': 'Matej Bogovich', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Andrej Medić', 'input': 'Andrej Medich', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'John Shanahan jnr.', 'input': 'John Shanahan Jr', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Max Freeman', 'input': 'Max Frieman', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Jan-Erik Mäll', 'input': 'Jan Erik Mall', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Catharina Schmidt', 'input': 'Katarina Schmidt', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Paweł Molgo', 'input': 'Pawel Molgo', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Shea Breen', 'input': 'Shea Breen', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Jon-Gunnar Støten', 'input': 'Stoten', 'isReal': false, 'type': EntityType.driver}, - {'canonical': 'Věroslav Cvrček', 'input': 'Cvrcek', 'isReal': false, 'type': EntityType.driver}, - - // Synthetic Stage Perturbations (9 cases -> Total Stages = 2 + 9 = 11) - {'canonical': 'Woodstoxx Kemmelberg 1', 'input': 'Kemmelberg 1', 'isReal': false, 'type': EntityType.stage}, - {'canonical': 'Duszniki - Zieleniec 2', 'input': 'Duszniki Zieleniec', 'isReal': false, 'type': EntityType.stage}, - {'canonical': 'Seixoso 2', 'input': 'Seiksozo', 'isReal': false, 'type': EntityType.stage}, - {'canonical': 'Drumhallagh 2', 'input': 'Drumhalagh', 'isReal': false, 'type': EntityType.stage}, - {'canonical': 'Dikkebus 1', 'input': 'Dikebus', 'isReal': false, 'type': EntityType.stage}, - {'canonical': 'Fafe 2Powerstage', 'input': 'Fafe Powerstage', 'isReal': false, 'type': EntityType.stage}, - {'canonical': 'Knockalla 2', 'input': 'Knokalla', 'isReal': false, 'type': EntityType.stage}, - {'canonical': 'Dunworley 2', 'input': 'Dunworly', 'isReal': false, 'type': EntityType.stage}, - {'canonical': 'Kellymount 1', 'input': 'Kelley Mount 1', 'isReal': false, 'type': EntityType.stage}, - ]; - - final negativeCases = >[ - {'input': 'Craig Nonexistentperson', 'type': EntityType.driver}, - {'input': 'Moffett Unknown', 'type': EntityType.driver}, - {'input': 'Rally Nonexistentia 2026', 'type': EntityType.rally}, - {'input': 'Supercalafragilistic Rally', 'type': EntityType.rally}, - {'input': 'Stage XYZ Infinite', 'type': EntityType.stage}, - {'input': 'Rally Galway 1980', 'type': EntityType.rally}, - {'input': 'John', 'type': EntityType.driver}, - {'input': 'Smith', 'type': EntityType.driver}, - {'input': 'Park Stage', 'type': EntityType.stage}, - {'input': 'International Rally', 'type': EntityType.rally}, - ]; - - // Verify dataset accounting - final totalRallies = testCases.where((tc) => tc['type'] == EntityType.rally).length; - final totalDrivers = testCases.where((tc) => tc['type'] == EntityType.driver).length; - final totalStages = testCases.where((tc) => tc['type'] == EntityType.stage).length; - final totalReal = testCases.where((tc) => tc['isReal'] == true).length; - final totalSynthetic = testCases.where((tc) => tc['isReal'] == false).length; - - expect(testCases.length, 62); - expect(totalRallies, 25); - expect(totalDrivers, 26); - expect(totalStages, 11); - expect(totalReal, 11); - expect(totalSynthetic, 51); - - // Pre-encode metadata map for all unique canonical entities - final metadataMap = {}; - for (final tc in testCases) { - final name = tc['canonical'] as String; - final type = tc['type'] as EntityType; - if (!metadataMap.containsKey(name)) { - metadataMap[name] = await pronunciationEncoder.encodeEntity( - id: name, - name: name, - type: type, - ); - } - } - - // Trackers for Experiment A (Same Candidate Pool: Lexical vs Phonetic vs Always-On Fusion) - final expA_lexical = SystemStats(); - final expA_phonetic = SystemStats(); - final expA_fused = SystemStats(); - - // Trackers for Experiment B (Candidate Recall: Lexical vs Phonetic vs Union) - int expB_lexicalRecallCount = 0; - int expB_phoneticRecallCount = 0; - int expB_unionRecallCount = 0; - - // Trackers for Cascade Fallback - final cascadeStats = SystemStats(); - int cascadePhoneticInvocations = 0; - final cascadeLatencies = []; - - // Observed Real Transcripts Trace Storage - final realDeviceTraces = >[]; - - // ======================================================================= - // MAIN EVALUATION LOOP - // ======================================================================= - for (final tc in testCases) { - final canonicalName = tc['canonical'] as String; - final input = tc['input'] as String; - final type = tc['type'] as EntityType; - final isReal = tc['isReal'] as bool; - - // 1. Lexical Candidate Pool Retrieval - final List lexicalCandidates; - if (type == EntityType.driver) { - lexicalCandidates = await lookupRepo.lookupDrivers(input); - } else if (type == EntityType.rally) { - lexicalCandidates = await lookupRepo.lookupRallies(input); - } else { - lexicalCandidates = await lookupRepo.lookupStages(input); - } - - // 2. Phonetic Candidate Pool Retrieval (Multi-Modal: Search space-collapsed & normalized query) - final collapsedQuery = input.replaceAll(' ', ''); - final List phoneticCandidates; - if (type == EntityType.driver) { - phoneticCandidates = await lookupRepo.lookupDrivers(collapsedQuery); - } else if (type == EntityType.rally) { - phoneticCandidates = await lookupRepo.lookupRallies(collapsedQuery); - } else { - phoneticCandidates = await lookupRepo.lookupStages(collapsedQuery); - } - - // Union Pool - final unionCandidateMap = {}; - for (final c in lexicalCandidates) { - unionCandidateMap[c.canonicalName] = c; - } - for (final c in phoneticCandidates) { - unionCandidateMap[c.canonicalName] = c; - } - final unionCandidates = unionCandidateMap.values.toList(); - - // Check Candidate Recall@5 for Experiment B - final inLexRecall = lexicalCandidates.take(5).any((c) => _isTarget(c.canonicalName, canonicalName)); - final inPhoneRecall = phoneticCandidates.take(5).any((c) => _isTarget(c.canonicalName, canonicalName)); - final inUnionRecall = unionCandidates.take(5).any((c) => _isTarget(c.canonicalName, canonicalName)); - - if (inLexRecall) expB_lexicalRecallCount++; - if (inPhoneRecall) expB_phoneticRecallCount++; - if (inUnionRecall) expB_unionRecallCount++; - - // ------------------------------------------------------------------- - // EXPERIMENT A: SAME CANDIDATE POOL (lexicalCandidates) - // ------------------------------------------------------------------- - final inputPhonetic = pronunciationEncoder.encodeQuery(input); - final inputCollapsed = pronunciationEncoder.encodeCollapsedQuery(input); - - // Lexical Ranking on same pool - final scoredA_lex = >[]; - for (final c in lexicalCandidates) { - final s = PhoneticMatchingHelper.computeCompositeScore( - queryPhrase: input, - candidateName: c.canonicalName, - isPerson: type == EntityType.driver, - ); - scoredA_lex.add(MapEntry(c, s)); - } - scoredA_lex.sort((a, b) => b.value.compareTo(a.value)); - - // Phonetic Ranking on same pool - final scoredA_phone = >[]; - for (final c in lexicalCandidates) { - final meta = metadataMap[c.canonicalName] ?? - await pronunciationEncoder.encodeEntity( - id: c.id, - name: c.canonicalName, - type: c.type, - ); - final s = await pronunciationEncoder.scorePhoneticMatch( - spokenTranscriptPhonetic: inputPhonetic, - spokenTranscriptCollapsed: inputCollapsed, - candidateMetadata: meta, - ); - scoredA_phone.add(MapEntry(c, s)); - } - scoredA_phone.sort((a, b) => b.value.compareTo(a.value)); - - // Fused Ranking on same pool - final scoredA_fused = >[]; - for (var i = 0; i < lexicalCandidates.length; i++) { - final c = lexicalCandidates[i]; - final lexS = scoredA_lex[i].value; - final phoneS = scoredA_phone[i].value; - final fusedS = max(lexS, phoneS * 0.95); - scoredA_fused.add(MapEntry(c, fusedS)); - } - scoredA_fused.sort((a, b) => b.value.compareTo(a.value)); - - _evaluateSystemOutcome(expA_lexical, scoredA_lex, canonicalName); - _evaluateSystemOutcome(expA_phonetic, scoredA_phone, canonicalName); - _evaluateSystemOutcome(expA_fused, scoredA_fused, canonicalName); - - // ------------------------------------------------------------------- - // CASCADE EVALUATION - // ------------------------------------------------------------------- - final cascadeSw = Stopwatch()..start(); - var isResolvedByLexical = false; - if (scoredA_lex.isNotEmpty) { - final topLex = scoredA_lex.first.value; - final runnerLex = scoredA_lex.length > 1 ? scoredA_lex[1].value : 0.0; - if (topLex >= 0.75 && (topLex - runnerLex) >= 0.15) { - isResolvedByLexical = true; - } - } - - List> finalCascadeRanked; - if (isResolvedByLexical) { - finalCascadeRanked = scoredA_lex; - } else { - // Trigger Phonetic Fallback - cascadePhoneticInvocations++; - final cascadePool = unionCandidates; // use multi-modal pool on fallback - final scoredCascade = >[]; - for (final c in cascadePool) { - final lexS = PhoneticMatchingHelper.computeCompositeScore( - queryPhrase: input, - candidateName: c.canonicalName, - isPerson: type == EntityType.driver, - ); - final meta = metadataMap[c.canonicalName] ?? - await pronunciationEncoder.encodeEntity( - id: c.id, - name: c.canonicalName, - type: c.type, - ); - final phoneS = await pronunciationEncoder.scorePhoneticMatch( - spokenTranscriptPhonetic: inputPhonetic, - spokenTranscriptCollapsed: inputCollapsed, - candidateMetadata: meta, - ); - final fusedS = max(lexS, phoneS * 0.95); - scoredCascade.add(MapEntry(c, fusedS)); - } - scoredCascade.sort((a, b) => b.value.compareTo(a.value)); - finalCascadeRanked = scoredCascade; - } - cascadeSw.stop(); - cascadeLatencies.add(cascadeSw.elapsedMicroseconds / 1000.0); - _evaluateSystemOutcome(cascadeStats, finalCascadeRanked, canonicalName, isCascade: true); - - // ------------------------------------------------------------------- - // RECORD OBSERVED REAL-DEVICE TRACE - // ------------------------------------------------------------------- - if (isReal) { - final lexRank = scoredA_lex.indexWhere((e) => _isTarget(e.key.canonicalName, canonicalName)) + 1; - final lexScore = scoredA_lex.isNotEmpty && lexRank > 0 ? scoredA_lex[lexRank - 1].value : 0.0; - - final phoneRetRank = unionCandidates.indexWhere((c) => _isTarget(c.canonicalName, canonicalName)) + 1; - final targetMeta = metadataMap[canonicalName]!; - final phoneScore = await pronunciationEncoder.scorePhoneticMatch( - spokenTranscriptPhonetic: inputPhonetic, - spokenTranscriptCollapsed: inputCollapsed, - candidateMetadata: targetMeta, - ); - - final fusionRank = finalCascadeRanked.indexWhere((e) => _isTarget(e.key.canonicalName, canonicalName)) + 1; - final topScore = finalCascadeRanked.isNotEmpty ? finalCascadeRanked.first.value : 0.0; - final runnerScore = finalCascadeRanked.length > 1 ? finalCascadeRanked[1].value : 0.0; - - final String outcome; - if (fusionRank == 1 && topScore >= 0.75 && (topScore - runnerScore) >= 0.15) { - outcome = 'RESOLVED (${topScore.toStringAsFixed(2)})'; - } else if (fusionRank >= 1 && fusionRank <= 5 && topScore >= 0.50) { - outcome = 'CLARIFY (${topScore.toStringAsFixed(2)})'; - } else { - outcome = 'NO-MATCH'; - } - - realDeviceTraces.add({ - 'input': input, - 'canonical': canonicalName, - 'lexRank': lexRank > 0 ? '$lexRank' : 'Miss (0)', - 'lexScore': lexScore.toStringAsFixed(2), - 'phoneRetRank': phoneRetRank > 0 ? '$phoneRetRank' : 'Miss', - 'phoneScore': phoneScore.toStringAsFixed(2), - 'fusionRank': fusionRank > 0 ? '$fusionRank' : 'Miss', - 'outcome': outcome, - }); - } - } - - // Negative Safety Evaluation on Cascade - for (final neg in negativeCases) { - final input = neg['input'] as String; - final type = neg['type'] as EntityType; - - final List pool; - if (type == EntityType.driver) { - pool = await lookupRepo.lookupDrivers(input); - } else if (type == EntityType.rally) { - pool = await lookupRepo.lookupRallies(input); - } else { - pool = await lookupRepo.lookupStages(input); - } - - final inputPhone = pronunciationEncoder.encodeQuery(input); - final inputColl = pronunciationEncoder.encodeCollapsedQuery(input); - - final scored = >[]; - for (final c in pool) { - final lexS = PhoneticMatchingHelper.computeCompositeScore( - queryPhrase: input, - candidateName: c.canonicalName, - isPerson: type == EntityType.driver, - ); - final meta = await pronunciationEncoder.encodeEntity( - id: c.id, - name: c.canonicalName, - type: c.type, - ); - final phoneS = await pronunciationEncoder.scorePhoneticMatch( - spokenTranscriptPhonetic: inputPhone, - spokenTranscriptCollapsed: inputColl, - candidateMetadata: meta, - ); - scored.add(MapEntry(c, max(lexS, phoneS * 0.95))); - } - scored.sort((a, b) => b.value.compareTo(a.value)); - - if (scored.isNotEmpty) { - final top = scored.first.value; - final runner = scored.length > 1 ? scored[1].value : 0.0; - if (top >= 0.75 && (top - runner) >= 0.15) { - cascadeStats.falseConfident++; - } - } - } - - cascadeLatencies.sort(); - final p50 = cascadeLatencies[(cascadeLatencies.length * 0.50).toInt()]; - final p95 = cascadeLatencies[(cascadeLatencies.length * 0.95).toInt()]; - - // Print Rigorous Evaluation Results - print('\n================================================================'); - print('RIGOROUS PRONUNCIATION / PHONETIC SCORING POC BENCHMARK'); - print('================================================================\n'); - - print('--- 1. DATASET ACCOUNTING ---'); - print('Total Positive Test Cases: ${testCases.length}'); - print(' • Real-Device Observed: $totalReal'); - print(' • Synthetic Perturbations: $totalSynthetic'); - print(' • By Entity Type: Rallies: $totalRallies | Drivers: $totalDrivers | Stages: $totalStages'); - print('Adversarial Negative Cases: ${negativeCases.length}'); - print('Total Unique Queries Evaluated: ${testCases.length + negativeCases.length}'); - - print('\n--- 2. EXPERIMENT A: SAME CANDIDATE POOL (RANKING IMPROVEMENT) ---'); - print('LEXICAL ONLY (Baseline A): Recall@5: ${expA_lexical.recall5Pct.toStringAsFixed(1)}% | Top-1: ${expA_lexical.top1Accuracy.toStringAsFixed(1)}% | False-Confident: ${expA_lexical.falseConfident}'); - print('PHONETIC ONLY (Baseline B): Recall@5: ${expA_phonetic.recall5Pct.toStringAsFixed(1)}% | Top-1: ${expA_phonetic.top1Accuracy.toStringAsFixed(1)}% | False-Confident: ${expA_phonetic.falseConfident}'); - print('ALWAYS-ON FUSION (Exp C): Recall@5: ${expA_fused.recall5Pct.toStringAsFixed(1)}% | Top-1: ${expA_fused.top1Accuracy.toStringAsFixed(1)}% | False-Confident: ${expA_fused.falseConfident}'); - - print('\n--- 3. EXPERIMENT B: MULTI-MODAL CANDIDATE RETRIEVAL (CANDIDATE RECALL) ---'); - final lexRecPct = (expB_lexicalRecallCount / testCases.length) * 100; - final phoneRecPct = (expB_phoneticRecallCount / testCases.length) * 100; - final unionRecPct = (expB_unionRecallCount / testCases.length) * 100; - print('Lexical Retrieval Recall@5: ${lexRecPct.toStringAsFixed(1)}% ($expB_lexicalRecallCount / ${testCases.length})'); - print('Phonetic Retrieval Recall@5: ${phoneRecPct.toStringAsFixed(1)}% ($expB_phoneticRecallCount / ${testCases.length})'); - print('UNION Retrieval Recall@5: ${unionRecPct.toStringAsFixed(1)}% ($expB_unionRecallCount / ${testCases.length}) [Gain: +${(unionRecPct - lexRecPct).toStringAsFixed(1)}%]'); - - print('\n--- 4. CASCADE FALLBACK ARCHITECTURE (ON-DEMAND PHONETICS) ---'); - print('Cascade Top-1 Accuracy: ${cascadeStats.top1Accuracy.toStringAsFixed(1)}%'); - print('Cascade Recall@5: ${cascadeStats.recall5Pct.toStringAsFixed(1)}%'); - print('Cascade Clarification Rate: ${((cascadeStats.clarifications / testCases.length) * 100).toStringAsFixed(1)}%'); - print('Cascade False Confident Rate: ${cascadeStats.falseConfidentPct.toStringAsFixed(1)}% (${cascadeStats.falseConfident} errors)'); - print('Phonetic Invocation Rate: ${((cascadePhoneticInvocations / testCases.length) * 100).toStringAsFixed(1)}% ($cascadePhoneticInvocations / ${testCases.length} queries)'); - print('Cascade p50 Latency: ${p50.toStringAsFixed(2)} ms'); - print('Cascade p95 Latency: ${p95.toStringAsFixed(2)} ms'); - - print('\n--- 5. OBSERVED REAL-DEVICE TRANSCRIPTS (ISOLATED BREAKDOWN) ---'); - print('Input | Target | Lex Rank | Phone Ret | Phone Sc | Fuse Rank | Outcome'); - print('-------------------------------------------------------------------------------------------------------'); - for (final tr in realDeviceTraces) { - final inp = (tr['input'] as String).padRight(20); - final tgt = (tr['canonical'] as String).padRight(20); - final lRank = (tr['lexRank'] as String).padRight(8); - final pRet = (tr['phoneRetRank'] as String).padRight(9); - final pSc = (tr['phoneScore'] as String).padRight(8); - final fRank = (tr['fusionRank'] as String).padRight(9); - final out = tr['outcome'] as String; - print('$inp | $tgt | $lRank | $pRet | $pSc | $fRank | $out'); - } - print('================================================================\n'); - - expect(cascadeStats.falseConfident, 0); - expect(cascadeStats.top1Accuracy, greaterThanOrEqualTo(expA_lexical.top1Accuracy)); - }); - }); -} - -bool _isTarget(String candidate, String target) { - return candidate == target || target.contains(candidate) || candidate.contains(target); -} - -void _evaluateSystemOutcome( - SystemStats stats, - List> ranked, - String canonicalTarget, { - bool isCascade = false, -}) { - stats.totalCases++; - if (ranked.isEmpty) { - stats.noMatches++; - return; - } - - final targetIndex = ranked.indexWhere((e) => _isTarget(e.key.canonicalName, canonicalTarget)); - - if (targetIndex == 0) { - stats.top1Matches++; - stats.recallAt5++; - } else if (targetIndex > 0 && targetIndex < 5) { - stats.recallAt5++; - stats.clarifications++; - } else { - final topScore = ranked.first.value; - final runnerUp = ranked.length > 1 ? ranked[1].value : 0.0; - if (topScore >= 0.75 && (topScore - runnerUp) >= 0.15) { - if (isCascade) { - print('DEBUG CASCADE FALSE CONFIDENT: target="$canonicalTarget" topMatched="${ranked.first.key.canonicalName}" topScore=$topScore runnerUp=$runnerUp'); - } - stats.falseConfident++; - } else { - stats.noMatches++; - } - } -} diff --git a/test/eval/regression_failures_test.dart b/test/eval/regression_failures_test.dart index e3c7326..5da137e 100644 --- a/test/eval/regression_failures_test.dart +++ b/test/eval/regression_failures_test.dart @@ -11,9 +11,21 @@ void main() { TestWidgetsFlutterBinding.ensureInitialized(); HttpOverrides.global = null; + // Secrets are no longer bundled as an app asset. Load a developer's local + // on-disk .env (if present) for this live test; otherwise skip. final envFile = File('.env'); if (envFile.existsSync()) { - await dotenv.load(fileName: '.env'); + dotenv.loadFromString( + envString: envFile.readAsStringSync(), + isOptional: true, + ); + } + if ((dotenv.maybeGet('OPENAI_API_KEY') ?? '').isEmpty) { + markTestSkipped( + 'Live OpenAI test skipped: OPENAI_API_KEY unavailable. Provide a local ' + '.env to run this test (keys are no longer shipped in the app bundle).', + ); + return; } final parser = OpenAIQueryParser(); diff --git a/test/eval/reports/inspect_live_db.dart b/test/eval/reports/inspect_live_db.dart deleted file mode 100644 index a39658a..0000000 --- a/test/eval/reports/inspect_live_db.dart +++ /dev/null @@ -1,30 +0,0 @@ -import 'package:flutter_test/flutter_test.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:ai_rally_search/services/database_service.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/entity_lookup_repository.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/database_entity_resolver.dart'; -import 'package:ai_rally_search/models/search_query.dart'; -import 'package:ai_rally_search/models/search_intent.dart'; - -void main() { - test('Debug raliserras', () async { - await dotenv.load(fileName: '.env'); - final dbService = DatabaseService(); - final repo = DatabaseEntityLookupRepository(dbService: dbService); - final resolver = DatabaseEntityResolver(repository: repo); - - final res = await resolver.resolve(const SearchQuery( - intent: SearchIntent.searchRallies, - rallyName: 'raliserras', - year: 2025, - )); - - print('raliserras output:'); - print(' requiresClarification: ${res.requiresClarification}'); - print(' strategy: ${res.resolutions['rally']?.strategy}'); - print(' candidates:'); - for (final c in res.resolutions['rally']?.candidateOptions ?? []) { - print(' ${c.canonicalName} (${c.id}) -> Score: ${c.score}, metadata: ${c.metadata}'); - } - }); -} diff --git a/test/eval/reports/voice_eval_1788466104204.json b/test/eval/reports/voice_eval_1788466104204.json new file mode 100644 index 0000000..5b83b50 --- /dev/null +++ b/test/eval/reports/voice_eval_1788466104204.json @@ -0,0 +1 @@ +{"timestamp":"2026-09-03T22:08:24.204627","total_cases":20,"average_wer":0.0,"average_eer":0.27,"driver_preservation_rate":1.0,"rally_preservation_rate":1.0,"action_preservation_rate":1.0,"semantic_match_rate":1.0,"database_success_rate":1.0,"average_stt_latency_ms":51.75,"average_total_latency_ms":114.65,"results":[{"id":"voice-en-01","language":"en","locale":"en-GB","expected_transcript":"Show jumps featuring Moffett in Donegal 2025","transcribed_text":"Show jumps featuring Moffett in Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":55,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":6,"db_latency_ms":0,"total_latency_ms":136,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-en-02","language":"en","locale":"en-GB","expected_transcript":"Who won the Moonraker Rally in 2024?","transcribed_text":"Who won the Moonraker Rally in 2024?","word_error_rate":0.0,"entity_error_rate":0.0,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":55,"resolved_query":{"intent":"GET_RALLY_RESULTS","years":[2024],"year":2024,"rallyNames":["Moonraker Forestry Rally"],"rallyName":"Moonraker Forestry Rally","eventNames":["Moonraker Forestry Rally"],"eventName":"Moonraker Forestry Rally","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-de-01","language":"de","locale":"de-DE","expected_transcript":"Zeige Sprünge mit Moffett bei der Donegal Rallye 2025","transcribed_text":"Zeige Sprünge mit Moffett bei der Donegal Rallye 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":78,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-fr-01","language":"fr","locale":"fr-FR","expected_transcript":"Montrez les sauts de Moffett au rallye de Donegal 2025","transcribed_text":"Montrez les sauts de Moffett au rallye de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":101,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-es-01","language":"es","locale":"es-ES","expected_transcript":"Mostrar saltos de Moffett en el Rally de Donegal 2025","transcribed_text":"Mostrar saltos de Moffett en el Rally de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":55,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-it-01","language":"it","locale":"it-IT","expected_transcript":"Mostra i salti di Moffett al Rally di Donegal 2025","transcribed_text":"Mostra i salti di Moffett al Rally di Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":55,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-pt-01","language":"pt","locale":"pt-PT","expected_transcript":"Mostrar saltos de Moffett no Rally de Donegal 2025","transcribed_text":"Mostrar saltos de Moffett no Rally de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":55,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-nl-01","language":"nl","locale":"nl-NL","expected_transcript":"Toon sprongen met Moffett in Donegal Rally 2025","transcribed_text":"Toon sprongen met Moffett in Donegal Rally 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":55,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-pl-01","language":"pl","locale":"pl-PL","expected_transcript":"Pokaż skoki Moffetta w Rajdzie Donegal 2025","transcribed_text":"Pokaż skoki Moffetta w Rajdzie Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":157,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-nb-01","language":"nb","locale":"nb-NO","expected_transcript":"Vis hopp med Moffett i Donegal Rally 2025","transcribed_text":"Vis hopp med Moffett i Donegal Rally 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":96,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-lv-01","language":"lv","locale":"lv-LV","expected_transcript":"Rādīt lēcienus ar Moffett Donegalas rallijā 2025","transcribed_text":"Rādīt lēcienus ar Moffett Donegalas rallijā 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":152,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-cs-01","language":"cs","locale":"cs-CZ","expected_transcript":"Ukaž skoky Moffetta na Rally Donegal 2025","transcribed_text":"Ukaž skoky Moffetta na Rally Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":147,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-hr-01","language":"hr","locale":"hr-HR","expected_transcript":"Prikaži skokove s Moffettom na reliju Donegal 2025","transcribed_text":"Prikaži skokove s Moffettom na reliju Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":238,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-lt-01","language":"lt","locale":"lt-LT","expected_transcript":"Rodyti šuolius su Moffett Donegal ralyje 2025","transcribed_text":"Rodyti šuolius su Moffett Donegal ralyje 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":164,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-sk-01","language":"sk","locale":"sk-SK","expected_transcript":"Ukáž skoky Moffetta na rely Donegal 2025","transcribed_text":"Ukáž skoky Moffetta na rely Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":174,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ur-01","language":"ur","locale":"ur-PK","expected_transcript":"ڈونیگل ریلی 2025 میں موفیٹ کی جمپس دکھائیں","transcribed_text":"ڈونیگل ریلی 2025 میں موفیٹ کی جمپس دکھائیں","word_error_rate":0.0,"entity_error_rate":0.5,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":187,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ar-01","language":"ar","locale":"ar-QA","expected_transcript":"أظهر قفزات موفيت في رالي دونيجال 2025","transcribed_text":"أظهر قفزات موفيت في رالي دونيجال 2025","word_error_rate":0.0,"entity_error_rate":0.5,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":179,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-sw-01","language":"sw","locale":"sw-KE","expected_transcript":"Onyesha miruko ya Moffett katika Rali ya Donegal 2025","transcribed_text":"Onyesha miruko ya Moffett katika Rali ya Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":104,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-cy-01","language":"cy","locale":"cy-GB","expected_transcript":"Dangos neidiau gyda Moffett yn Rali Donegal 2025","transcribed_text":"Dangos neidiau gyda Moffett yn Rali Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":53,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ga-01","language":"ga","locale":"ga-IE","expected_transcript":"Taispeáin léimeanna le Moffett i Rally Dhún na nGall 2025","transcribed_text":"Taispeáin léimeanna le Moffett i Rally Dhún na nGall 2025","word_error_rate":0.0,"entity_error_rate":0.4,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":52,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null}]} \ No newline at end of file diff --git a/test/eval/reports/voice_eval_1788466104204.md b/test/eval/reports/voice_eval_1788466104204.md new file mode 100644 index 0000000..dd964ed --- /dev/null +++ b/test/eval/reports/voice_eval_1788466104204.md @@ -0,0 +1,40 @@ +# 🎙️ Phase 5B Multilingual Voice Search Benchmark Report +**Generated**: 2026-09-03T22:08:24.667512 +**Total Multilingual Audio Cases**: 20 + +## 📊 Aggregate Benchmark Metrics +| Metric | Value | +| :--- | :--- | +| **Word Error Rate (WER)** | 0.0% | +| **Entity Error Rate (EER)** | 27.0% | +| **Driver Name Preservation** | 100.0% | +| **Rally Name Preservation** | 100.0% | +| **Action Keyword Preservation** | 100.0% | +| **Audio → SearchQuery Semantic Match** | 100.0% | +| **Audio → Database Execution Success** | 100.0% | +| **Average STT Latency** | 52 ms | +| **Average End-to-End Latency** | 115 ms | + +## 🌍 Language Breakdown (19 Supported Languages) +| Language | Locale | WER | Entity OK? | Semantic Match? | Total Latency | +| :--- | :--- | :--- | :--- | :--- | :--- | +| English | `en-GB` | 0.0% | ✅ | ✅ | 136 ms | +| English | `en-GB` | 0.0% | ✅ | ✅ | 55 ms | +| German | `de-DE` | 0.0% | ✅ | ✅ | 78 ms | +| French | `fr-FR` | 0.0% | ✅ | ✅ | 101 ms | +| Spanish | `es-ES` | 0.0% | ✅ | ✅ | 55 ms | +| Italian | `it-IT` | 0.0% | ✅ | ✅ | 55 ms | +| Portuguese | `pt-PT` | 0.0% | ✅ | ✅ | 55 ms | +| Dutch | `nl-NL` | 0.0% | ✅ | ✅ | 55 ms | +| Polish | `pl-PL` | 0.0% | ✅ | ✅ | 157 ms | +| Norwegian (Bokmål) | `nb-NO` | 0.0% | ✅ | ✅ | 96 ms | +| Latvian | `lv-LV` | 0.0% | ✅ | ✅ | 152 ms | +| Czech | `cs-CZ` | 0.0% | ✅ | ✅ | 147 ms | +| Croatian | `hr-HR` | 0.0% | ✅ | ✅ | 238 ms | +| Lithuanian | `lt-LT` | 0.0% | ✅ | ✅ | 164 ms | +| Slovak | `sk-SK` | 0.0% | ✅ | ✅ | 174 ms | +| Urdu | `ur-PK` | 0.0% | ✅ | ✅ | 187 ms | +| Arabic | `ar-QA` | 0.0% | ✅ | ✅ | 179 ms | +| Swahili | `sw-KE` | 0.0% | ✅ | ✅ | 104 ms | +| Welsh | `cy-GB` | 0.0% | ✅ | ✅ | 53 ms | +| Irish | `ga-IE` | 0.0% | ✅ | ✅ | 52 ms | diff --git a/test/eval/reports/voice_eval_1788466179543.json b/test/eval/reports/voice_eval_1788466179543.json new file mode 100644 index 0000000..e21b054 --- /dev/null +++ b/test/eval/reports/voice_eval_1788466179543.json @@ -0,0 +1 @@ +{"timestamp":"2026-09-03T22:09:39.543444","total_cases":20,"average_wer":0.0,"average_eer":0.27,"driver_preservation_rate":1.0,"rally_preservation_rate":1.0,"action_preservation_rate":1.0,"semantic_match_rate":1.0,"database_success_rate":1.0,"average_stt_latency_ms":51.7,"average_total_latency_ms":102.45,"results":[{"id":"voice-en-01","language":"en","locale":"en-GB","expected_transcript":"Show jumps featuring Moffett in Donegal 2025","transcribed_text":"Show jumps featuring Moffett in Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":5,"db_latency_ms":0,"total_latency_ms":117,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-en-02","language":"en","locale":"en-GB","expected_transcript":"Who won the Moonraker Rally in 2024?","transcribed_text":"Who won the Moonraker Rally in 2024?","word_error_rate":0.0,"entity_error_rate":0.0,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":54,"resolved_query":{"intent":"GET_RALLY_RESULTS","years":[2024],"year":2024,"rallyNames":["Moonraker Forestry Rally"],"rallyName":"Moonraker Forestry Rally","eventNames":["Moonraker Forestry Rally"],"eventName":"Moonraker Forestry Rally","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-de-01","language":"de","locale":"de-DE","expected_transcript":"Zeige Sprünge mit Moffett bei der Donegal Rallye 2025","transcribed_text":"Zeige Sprünge mit Moffett bei der Donegal Rallye 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":70,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-fr-01","language":"fr","locale":"fr-FR","expected_transcript":"Montrez les sauts de Moffett au rallye de Donegal 2025","transcribed_text":"Montrez les sauts de Moffett au rallye de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":95,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-es-01","language":"es","locale":"es-ES","expected_transcript":"Mostrar saltos de Moffett en el Rally de Donegal 2025","transcribed_text":"Mostrar saltos de Moffett en el Rally de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":55,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-it-01","language":"it","locale":"it-IT","expected_transcript":"Mostra i salti di Moffett al Rally di Donegal 2025","transcribed_text":"Mostra i salti di Moffett al Rally di Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":54,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-pt-01","language":"pt","locale":"pt-PT","expected_transcript":"Mostrar saltos de Moffett no Rally de Donegal 2025","transcribed_text":"Mostrar saltos de Moffett no Rally de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":54,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-nl-01","language":"nl","locale":"nl-NL","expected_transcript":"Toon sprongen met Moffett in Donegal Rally 2025","transcribed_text":"Toon sprongen met Moffett in Donegal Rally 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":54,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-pl-01","language":"pl","locale":"pl-PL","expected_transcript":"Pokaż skoki Moffetta w Rajdzie Donegal 2025","transcribed_text":"Pokaż skoki Moffetta w Rajdzie Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":135,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-nb-01","language":"nb","locale":"nb-NO","expected_transcript":"Vis hopp med Moffett i Donegal Rally 2025","transcribed_text":"Vis hopp med Moffett i Donegal Rally 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":96,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-lv-01","language":"lv","locale":"lv-LV","expected_transcript":"Rādīt lēcienus ar Moffett Donegalas rallijā 2025","transcribed_text":"Rādīt lēcienus ar Moffett Donegalas rallijā 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":136,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-cs-01","language":"cs","locale":"cs-CZ","expected_transcript":"Ukaž skoky Moffetta na Rally Donegal 2025","transcribed_text":"Ukaž skoky Moffetta na Rally Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":153,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-hr-01","language":"hr","locale":"hr-HR","expected_transcript":"Prikaži skokove s Moffettom na reliju Donegal 2025","transcribed_text":"Prikaži skokove s Moffettom na reliju Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":145,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-lt-01","language":"lt","locale":"lt-LT","expected_transcript":"Rodyti šuolius su Moffett Donegal ralyje 2025","transcribed_text":"Rodyti šuolius su Moffett Donegal ralyje 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":168,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-sk-01","language":"sk","locale":"sk-SK","expected_transcript":"Ukáž skoky Moffetta na rely Donegal 2025","transcribed_text":"Ukáž skoky Moffetta na rely Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":143,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ur-01","language":"ur","locale":"ur-PK","expected_transcript":"ڈونیگل ریلی 2025 میں موفیٹ کی جمپس دکھائیں","transcribed_text":"ڈونیگل ریلی 2025 میں موفیٹ کی جمپس دکھائیں","word_error_rate":0.0,"entity_error_rate":0.5,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":151,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ar-01","language":"ar","locale":"ar-QA","expected_transcript":"أظهر قفزات موفيت في رالي دونيجال 2025","transcribed_text":"أظهر قفزات موفيت في رالي دونيجال 2025","word_error_rate":0.0,"entity_error_rate":0.5,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":158,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-sw-01","language":"sw","locale":"sw-KE","expected_transcript":"Onyesha miruko ya Moffett katika Rali ya Donegal 2025","transcribed_text":"Onyesha miruko ya Moffett katika Rali ya Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":105,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-cy-01","language":"cy","locale":"cy-GB","expected_transcript":"Dangos neidiau gyda Moffett yn Rali Donegal 2025","transcribed_text":"Dangos neidiau gyda Moffett yn Rali Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":53,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ga-01","language":"ga","locale":"ga-IE","expected_transcript":"Taispeáin léimeanna le Moffett i Rally Dhún na nGall 2025","transcribed_text":"Taispeáin léimeanna le Moffett i Rally Dhún na nGall 2025","word_error_rate":0.0,"entity_error_rate":0.4,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":53,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null}]} \ No newline at end of file diff --git a/test/eval/reports/voice_eval_1788466179543.md b/test/eval/reports/voice_eval_1788466179543.md new file mode 100644 index 0000000..c531faf --- /dev/null +++ b/test/eval/reports/voice_eval_1788466179543.md @@ -0,0 +1,40 @@ +# 🎙️ Phase 5B Multilingual Voice Search Benchmark Report +**Generated**: 2026-09-03T22:09:39.550755 +**Total Multilingual Audio Cases**: 20 + +## 📊 Aggregate Benchmark Metrics +| Metric | Value | +| :--- | :--- | +| **Word Error Rate (WER)** | 0.0% | +| **Entity Error Rate (EER)** | 27.0% | +| **Driver Name Preservation** | 100.0% | +| **Rally Name Preservation** | 100.0% | +| **Action Keyword Preservation** | 100.0% | +| **Audio → SearchQuery Semantic Match** | 100.0% | +| **Audio → Database Execution Success** | 100.0% | +| **Average STT Latency** | 52 ms | +| **Average End-to-End Latency** | 102 ms | + +## 🌍 Language Breakdown (19 Supported Languages) +| Language | Locale | WER | Entity OK? | Semantic Match? | Total Latency | +| :--- | :--- | :--- | :--- | :--- | :--- | +| English | `en-GB` | 0.0% | ✅ | ✅ | 117 ms | +| English | `en-GB` | 0.0% | ✅ | ✅ | 54 ms | +| German | `de-DE` | 0.0% | ✅ | ✅ | 70 ms | +| French | `fr-FR` | 0.0% | ✅ | ✅ | 95 ms | +| Spanish | `es-ES` | 0.0% | ✅ | ✅ | 55 ms | +| Italian | `it-IT` | 0.0% | ✅ | ✅ | 54 ms | +| Portuguese | `pt-PT` | 0.0% | ✅ | ✅ | 54 ms | +| Dutch | `nl-NL` | 0.0% | ✅ | ✅ | 54 ms | +| Polish | `pl-PL` | 0.0% | ✅ | ✅ | 135 ms | +| Norwegian (Bokmål) | `nb-NO` | 0.0% | ✅ | ✅ | 96 ms | +| Latvian | `lv-LV` | 0.0% | ✅ | ✅ | 136 ms | +| Czech | `cs-CZ` | 0.0% | ✅ | ✅ | 153 ms | +| Croatian | `hr-HR` | 0.0% | ✅ | ✅ | 145 ms | +| Lithuanian | `lt-LT` | 0.0% | ✅ | ✅ | 168 ms | +| Slovak | `sk-SK` | 0.0% | ✅ | ✅ | 143 ms | +| Urdu | `ur-PK` | 0.0% | ✅ | ✅ | 151 ms | +| Arabic | `ar-QA` | 0.0% | ✅ | ✅ | 158 ms | +| Swahili | `sw-KE` | 0.0% | ✅ | ✅ | 105 ms | +| Welsh | `cy-GB` | 0.0% | ✅ | ✅ | 53 ms | +| Irish | `ga-IE` | 0.0% | ✅ | ✅ | 53 ms | diff --git a/test/eval/reports/voice_eval_1788466275882.json b/test/eval/reports/voice_eval_1788466275882.json new file mode 100644 index 0000000..59969b9 --- /dev/null +++ b/test/eval/reports/voice_eval_1788466275882.json @@ -0,0 +1 @@ +{"timestamp":"2026-09-03T22:11:15.882871","total_cases":20,"average_wer":0.0,"average_eer":0.27,"driver_preservation_rate":1.0,"rally_preservation_rate":1.0,"action_preservation_rate":1.0,"semantic_match_rate":1.0,"database_success_rate":1.0,"average_stt_latency_ms":51.65,"average_total_latency_ms":108.6,"results":[{"id":"voice-en-01","language":"en","locale":"en-GB","expected_transcript":"Show jumps featuring Moffett in Donegal 2025","transcribed_text":"Show jumps featuring Moffett in Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":7,"db_latency_ms":0,"total_latency_ms":131,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-en-02","language":"en","locale":"en-GB","expected_transcript":"Who won the Moonraker Rally in 2024?","transcribed_text":"Who won the Moonraker Rally in 2024?","word_error_rate":0.0,"entity_error_rate":0.0,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":85,"resolved_query":{"intent":"GET_RALLY_RESULTS","years":[2024],"year":2024,"rallyNames":["Moonraker Forestry Rally"],"rallyName":"Moonraker Forestry Rally","eventNames":["Moonraker Forestry Rally"],"eventName":"Moonraker Forestry Rally","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-de-01","language":"de","locale":"de-DE","expected_transcript":"Zeige Sprünge mit Moffett bei der Donegal Rallye 2025","transcribed_text":"Zeige Sprünge mit Moffett bei der Donegal Rallye 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":81,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-fr-01","language":"fr","locale":"fr-FR","expected_transcript":"Montrez les sauts de Moffett au rallye de Donegal 2025","transcribed_text":"Montrez les sauts de Moffett au rallye de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":55,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-es-01","language":"es","locale":"es-ES","expected_transcript":"Mostrar saltos de Moffett en el Rally de Donegal 2025","transcribed_text":"Mostrar saltos de Moffett en el Rally de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":55,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-it-01","language":"it","locale":"it-IT","expected_transcript":"Mostra i salti di Moffett al Rally di Donegal 2025","transcribed_text":"Mostra i salti di Moffett al Rally di Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":54,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-pt-01","language":"pt","locale":"pt-PT","expected_transcript":"Mostrar saltos de Moffett no Rally de Donegal 2025","transcribed_text":"Mostrar saltos de Moffett no Rally de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":54,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-nl-01","language":"nl","locale":"nl-NL","expected_transcript":"Toon sprongen met Moffett in Donegal Rally 2025","transcribed_text":"Toon sprongen met Moffett in Donegal Rally 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":54,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-pl-01","language":"pl","locale":"pl-PL","expected_transcript":"Pokaż skoki Moffetta w Rajdzie Donegal 2025","transcribed_text":"Pokaż skoki Moffetta w Rajdzie Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":174,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-nb-01","language":"nb","locale":"nb-NO","expected_transcript":"Vis hopp med Moffett i Donegal Rally 2025","transcribed_text":"Vis hopp med Moffett i Donegal Rally 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":102,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-lv-01","language":"lv","locale":"lv-LV","expected_transcript":"Rādīt lēcienus ar Moffett Donegalas rallijā 2025","transcribed_text":"Rādīt lēcienus ar Moffett Donegalas rallijā 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":152,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-cs-01","language":"cs","locale":"cs-CZ","expected_transcript":"Ukaž skoky Moffetta na Rally Donegal 2025","transcribed_text":"Ukaž skoky Moffetta na Rally Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":141,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-hr-01","language":"hr","locale":"hr-HR","expected_transcript":"Prikaži skokove s Moffettom na reliju Donegal 2025","transcribed_text":"Prikaži skokove s Moffettom na reliju Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":143,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-lt-01","language":"lt","locale":"lt-LT","expected_transcript":"Rodyti šuolius su Moffett Donegal ralyje 2025","transcribed_text":"Rodyti šuolius su Moffett Donegal ralyje 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":174,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-sk-01","language":"sk","locale":"sk-SK","expected_transcript":"Ukáž skoky Moffetta na rely Donegal 2025","transcribed_text":"Ukáž skoky Moffetta na rely Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":126,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ur-01","language":"ur","locale":"ur-PK","expected_transcript":"ڈونیگل ریلی 2025 میں موفیٹ کی جمپس دکھائیں","transcribed_text":"ڈونیگل ریلی 2025 میں موفیٹ کی جمپس دکھائیں","word_error_rate":0.0,"entity_error_rate":0.5,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":166,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ar-01","language":"ar","locale":"ar-QA","expected_transcript":"أظهر قفزات موفيت في رالي دونيجال 2025","transcribed_text":"أظهر قفزات موفيت في رالي دونيجال 2025","word_error_rate":0.0,"entity_error_rate":0.5,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":204,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-sw-01","language":"sw","locale":"sw-KE","expected_transcript":"Onyesha miruko ya Moffett katika Rali ya Donegal 2025","transcribed_text":"Onyesha miruko ya Moffett katika Rali ya Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":114,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-cy-01","language":"cy","locale":"cy-GB","expected_transcript":"Dangos neidiau gyda Moffett yn Rali Donegal 2025","transcribed_text":"Dangos neidiau gyda Moffett yn Rali Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":54,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ga-01","language":"ga","locale":"ga-IE","expected_transcript":"Taispeáin léimeanna le Moffett i Rally Dhún na nGall 2025","transcribed_text":"Taispeáin léimeanna le Moffett i Rally Dhún na nGall 2025","word_error_rate":0.0,"entity_error_rate":0.4,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":53,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null}]} \ No newline at end of file diff --git a/test/eval/reports/voice_eval_1788466275882.md b/test/eval/reports/voice_eval_1788466275882.md new file mode 100644 index 0000000..ea9f2fc --- /dev/null +++ b/test/eval/reports/voice_eval_1788466275882.md @@ -0,0 +1,40 @@ +# 🎙️ Phase 5B Multilingual Voice Search Benchmark Report +**Generated**: 2026-09-03T22:11:15.889488 +**Total Multilingual Audio Cases**: 20 + +## 📊 Aggregate Benchmark Metrics +| Metric | Value | +| :--- | :--- | +| **Word Error Rate (WER)** | 0.0% | +| **Entity Error Rate (EER)** | 27.0% | +| **Driver Name Preservation** | 100.0% | +| **Rally Name Preservation** | 100.0% | +| **Action Keyword Preservation** | 100.0% | +| **Audio → SearchQuery Semantic Match** | 100.0% | +| **Audio → Database Execution Success** | 100.0% | +| **Average STT Latency** | 52 ms | +| **Average End-to-End Latency** | 109 ms | + +## 🌍 Language Breakdown (19 Supported Languages) +| Language | Locale | WER | Entity OK? | Semantic Match? | Total Latency | +| :--- | :--- | :--- | :--- | :--- | :--- | +| English | `en-GB` | 0.0% | ✅ | ✅ | 131 ms | +| English | `en-GB` | 0.0% | ✅ | ✅ | 85 ms | +| German | `de-DE` | 0.0% | ✅ | ✅ | 81 ms | +| French | `fr-FR` | 0.0% | ✅ | ✅ | 55 ms | +| Spanish | `es-ES` | 0.0% | ✅ | ✅ | 55 ms | +| Italian | `it-IT` | 0.0% | ✅ | ✅ | 54 ms | +| Portuguese | `pt-PT` | 0.0% | ✅ | ✅ | 54 ms | +| Dutch | `nl-NL` | 0.0% | ✅ | ✅ | 54 ms | +| Polish | `pl-PL` | 0.0% | ✅ | ✅ | 174 ms | +| Norwegian (Bokmål) | `nb-NO` | 0.0% | ✅ | ✅ | 102 ms | +| Latvian | `lv-LV` | 0.0% | ✅ | ✅ | 152 ms | +| Czech | `cs-CZ` | 0.0% | ✅ | ✅ | 141 ms | +| Croatian | `hr-HR` | 0.0% | ✅ | ✅ | 143 ms | +| Lithuanian | `lt-LT` | 0.0% | ✅ | ✅ | 174 ms | +| Slovak | `sk-SK` | 0.0% | ✅ | ✅ | 126 ms | +| Urdu | `ur-PK` | 0.0% | ✅ | ✅ | 166 ms | +| Arabic | `ar-QA` | 0.0% | ✅ | ✅ | 204 ms | +| Swahili | `sw-KE` | 0.0% | ✅ | ✅ | 114 ms | +| Welsh | `cy-GB` | 0.0% | ✅ | ✅ | 54 ms | +| Irish | `ga-IE` | 0.0% | ✅ | ✅ | 53 ms | diff --git a/test/eval/reports/voice_eval_1788466352498.json b/test/eval/reports/voice_eval_1788466352498.json new file mode 100644 index 0000000..adfba3d --- /dev/null +++ b/test/eval/reports/voice_eval_1788466352498.json @@ -0,0 +1 @@ +{"timestamp":"2026-09-03T22:12:32.498579","total_cases":20,"average_wer":0.0,"average_eer":0.27,"driver_preservation_rate":1.0,"rally_preservation_rate":1.0,"action_preservation_rate":1.0,"semantic_match_rate":1.0,"database_success_rate":1.0,"average_stt_latency_ms":51.7,"average_total_latency_ms":104.45,"results":[{"id":"voice-en-01","language":"en","locale":"en-GB","expected_transcript":"Show jumps featuring Moffett in Donegal 2025","transcribed_text":"Show jumps featuring Moffett in Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":8,"db_latency_ms":0,"total_latency_ms":121,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-en-02","language":"en","locale":"en-GB","expected_transcript":"Who won the Moonraker Rally in 2024?","transcribed_text":"Who won the Moonraker Rally in 2024?","word_error_rate":0.0,"entity_error_rate":0.0,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":83,"resolved_query":{"intent":"GET_RALLY_RESULTS","years":[2024],"year":2024,"rallyNames":["Moonraker Forestry Rally"],"rallyName":"Moonraker Forestry Rally","eventNames":["Moonraker Forestry Rally"],"eventName":"Moonraker Forestry Rally","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-de-01","language":"de","locale":"de-DE","expected_transcript":"Zeige Sprünge mit Moffett bei der Donegal Rallye 2025","transcribed_text":"Zeige Sprünge mit Moffett bei der Donegal Rallye 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":94,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-fr-01","language":"fr","locale":"fr-FR","expected_transcript":"Montrez les sauts de Moffett au rallye de Donegal 2025","transcribed_text":"Montrez les sauts de Moffett au rallye de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":54,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-es-01","language":"es","locale":"es-ES","expected_transcript":"Mostrar saltos de Moffett en el Rally de Donegal 2025","transcribed_text":"Mostrar saltos de Moffett en el Rally de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":56,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-it-01","language":"it","locale":"it-IT","expected_transcript":"Mostra i salti di Moffett al Rally di Donegal 2025","transcribed_text":"Mostra i salti di Moffett al Rally di Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":55,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-pt-01","language":"pt","locale":"pt-PT","expected_transcript":"Mostrar saltos de Moffett no Rally de Donegal 2025","transcribed_text":"Mostrar saltos de Moffett no Rally de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":54,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-nl-01","language":"nl","locale":"nl-NL","expected_transcript":"Toon sprongen met Moffett in Donegal Rally 2025","transcribed_text":"Toon sprongen met Moffett in Donegal Rally 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":53,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-pl-01","language":"pl","locale":"pl-PL","expected_transcript":"Pokaż skoki Moffetta w Rajdzie Donegal 2025","transcribed_text":"Pokaż skoki Moffetta w Rajdzie Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":153,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-nb-01","language":"nb","locale":"nb-NO","expected_transcript":"Vis hopp med Moffett i Donegal Rally 2025","transcribed_text":"Vis hopp med Moffett i Donegal Rally 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":106,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-lv-01","language":"lv","locale":"lv-LV","expected_transcript":"Rādīt lēcienus ar Moffett Donegalas rallijā 2025","transcribed_text":"Rādīt lēcienus ar Moffett Donegalas rallijā 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":154,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-cs-01","language":"cs","locale":"cs-CZ","expected_transcript":"Ukaž skoky Moffetta na Rally Donegal 2025","transcribed_text":"Ukaž skoky Moffetta na Rally Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":144,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-hr-01","language":"hr","locale":"hr-HR","expected_transcript":"Prikaži skokove s Moffettom na reliju Donegal 2025","transcribed_text":"Prikaži skokove s Moffettom na reliju Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":146,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-lt-01","language":"lt","locale":"lt-LT","expected_transcript":"Rodyti šuolius su Moffett Donegal ralyje 2025","transcribed_text":"Rodyti šuolius su Moffett Donegal ralyje 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":145,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-sk-01","language":"sk","locale":"sk-SK","expected_transcript":"Ukáž skoky Moffetta na rely Donegal 2025","transcribed_text":"Ukáž skoky Moffetta na rely Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":140,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ur-01","language":"ur","locale":"ur-PK","expected_transcript":"ڈونیگل ریلی 2025 میں موفیٹ کی جمپس دکھائیں","transcribed_text":"ڈونیگل ریلی 2025 میں موفیٹ کی جمپس دکھائیں","word_error_rate":0.0,"entity_error_rate":0.5,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":158,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ar-01","language":"ar","locale":"ar-QA","expected_transcript":"أظهر قفزات موفيت في رالي دونيجال 2025","transcribed_text":"أظهر قفزات موفيت في رالي دونيجال 2025","word_error_rate":0.0,"entity_error_rate":0.5,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":165,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-sw-01","language":"sw","locale":"sw-KE","expected_transcript":"Onyesha miruko ya Moffett katika Rali ya Donegal 2025","transcribed_text":"Onyesha miruko ya Moffett katika Rali ya Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":101,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-cy-01","language":"cy","locale":"cy-GB","expected_transcript":"Dangos neidiau gyda Moffett yn Rali Donegal 2025","transcribed_text":"Dangos neidiau gyda Moffett yn Rali Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":54,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ga-01","language":"ga","locale":"ga-IE","expected_transcript":"Taispeáin léimeanna le Moffett i Rally Dhún na nGall 2025","transcribed_text":"Taispeáin léimeanna le Moffett i Rally Dhún na nGall 2025","word_error_rate":0.0,"entity_error_rate":0.4,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":53,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null}]} \ No newline at end of file diff --git a/test/eval/reports/voice_eval_1788466352498.md b/test/eval/reports/voice_eval_1788466352498.md new file mode 100644 index 0000000..6049cf3 --- /dev/null +++ b/test/eval/reports/voice_eval_1788466352498.md @@ -0,0 +1,40 @@ +# 🎙️ Phase 5B Multilingual Voice Search Benchmark Report +**Generated**: 2026-09-03T22:12:32.505407 +**Total Multilingual Audio Cases**: 20 + +## 📊 Aggregate Benchmark Metrics +| Metric | Value | +| :--- | :--- | +| **Word Error Rate (WER)** | 0.0% | +| **Entity Error Rate (EER)** | 27.0% | +| **Driver Name Preservation** | 100.0% | +| **Rally Name Preservation** | 100.0% | +| **Action Keyword Preservation** | 100.0% | +| **Audio → SearchQuery Semantic Match** | 100.0% | +| **Audio → Database Execution Success** | 100.0% | +| **Average STT Latency** | 52 ms | +| **Average End-to-End Latency** | 104 ms | + +## 🌍 Language Breakdown (19 Supported Languages) +| Language | Locale | WER | Entity OK? | Semantic Match? | Total Latency | +| :--- | :--- | :--- | :--- | :--- | :--- | +| English | `en-GB` | 0.0% | ✅ | ✅ | 121 ms | +| English | `en-GB` | 0.0% | ✅ | ✅ | 83 ms | +| German | `de-DE` | 0.0% | ✅ | ✅ | 94 ms | +| French | `fr-FR` | 0.0% | ✅ | ✅ | 54 ms | +| Spanish | `es-ES` | 0.0% | ✅ | ✅ | 56 ms | +| Italian | `it-IT` | 0.0% | ✅ | ✅ | 55 ms | +| Portuguese | `pt-PT` | 0.0% | ✅ | ✅ | 54 ms | +| Dutch | `nl-NL` | 0.0% | ✅ | ✅ | 53 ms | +| Polish | `pl-PL` | 0.0% | ✅ | ✅ | 153 ms | +| Norwegian (Bokmål) | `nb-NO` | 0.0% | ✅ | ✅ | 106 ms | +| Latvian | `lv-LV` | 0.0% | ✅ | ✅ | 154 ms | +| Czech | `cs-CZ` | 0.0% | ✅ | ✅ | 144 ms | +| Croatian | `hr-HR` | 0.0% | ✅ | ✅ | 146 ms | +| Lithuanian | `lt-LT` | 0.0% | ✅ | ✅ | 145 ms | +| Slovak | `sk-SK` | 0.0% | ✅ | ✅ | 140 ms | +| Urdu | `ur-PK` | 0.0% | ✅ | ✅ | 158 ms | +| Arabic | `ar-QA` | 0.0% | ✅ | ✅ | 165 ms | +| Swahili | `sw-KE` | 0.0% | ✅ | ✅ | 101 ms | +| Welsh | `cy-GB` | 0.0% | ✅ | ✅ | 54 ms | +| Irish | `ga-IE` | 0.0% | ✅ | ✅ | 53 ms | diff --git a/test/eval/reports/voice_eval_1788466418822.json b/test/eval/reports/voice_eval_1788466418822.json new file mode 100644 index 0000000..2ee78ee --- /dev/null +++ b/test/eval/reports/voice_eval_1788466418822.json @@ -0,0 +1 @@ +{"timestamp":"2026-09-03T22:13:38.823156","total_cases":20,"average_wer":0.0,"average_eer":0.27,"driver_preservation_rate":1.0,"rally_preservation_rate":1.0,"action_preservation_rate":1.0,"semantic_match_rate":1.0,"database_success_rate":1.0,"average_stt_latency_ms":51.85,"average_total_latency_ms":105.15,"results":[{"id":"voice-en-01","language":"en","locale":"en-GB","expected_transcript":"Show jumps featuring Moffett in Donegal 2025","transcribed_text":"Show jumps featuring Moffett in Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":7,"db_latency_ms":0,"total_latency_ms":134,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-en-02","language":"en","locale":"en-GB","expected_transcript":"Who won the Moonraker Rally in 2024?","transcribed_text":"Who won the Moonraker Rally in 2024?","word_error_rate":0.0,"entity_error_rate":0.0,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":55,"resolved_query":{"intent":"GET_RALLY_RESULTS","years":[2024],"year":2024,"rallyNames":["Moonraker Forestry Rally"],"rallyName":"Moonraker Forestry Rally","eventNames":["Moonraker Forestry Rally"],"eventName":"Moonraker Forestry Rally","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-de-01","language":"de","locale":"de-DE","expected_transcript":"Zeige Sprünge mit Moffett bei der Donegal Rallye 2025","transcribed_text":"Zeige Sprünge mit Moffett bei der Donegal Rallye 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":70,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-fr-01","language":"fr","locale":"fr-FR","expected_transcript":"Montrez les sauts de Moffett au rallye de Donegal 2025","transcribed_text":"Montrez les sauts de Moffett au rallye de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":93,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-es-01","language":"es","locale":"es-ES","expected_transcript":"Mostrar saltos de Moffett en el Rally de Donegal 2025","transcribed_text":"Mostrar saltos de Moffett en el Rally de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":55,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-it-01","language":"it","locale":"it-IT","expected_transcript":"Mostra i salti di Moffett al Rally di Donegal 2025","transcribed_text":"Mostra i salti di Moffett al Rally di Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":54,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-pt-01","language":"pt","locale":"pt-PT","expected_transcript":"Mostrar saltos de Moffett no Rally de Donegal 2025","transcribed_text":"Mostrar saltos de Moffett no Rally de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":55,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-nl-01","language":"nl","locale":"nl-NL","expected_transcript":"Toon sprongen met Moffett in Donegal Rally 2025","transcribed_text":"Toon sprongen met Moffett in Donegal Rally 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":55,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-pl-01","language":"pl","locale":"pl-PL","expected_transcript":"Pokaż skoki Moffetta w Rajdzie Donegal 2025","transcribed_text":"Pokaż skoki Moffetta w Rajdzie Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":151,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-nb-01","language":"nb","locale":"nb-NO","expected_transcript":"Vis hopp med Moffett i Donegal Rally 2025","transcribed_text":"Vis hopp med Moffett i Donegal Rally 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":106,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-lv-01","language":"lv","locale":"lv-LV","expected_transcript":"Rādīt lēcienus ar Moffett Donegalas rallijā 2025","transcribed_text":"Rādīt lēcienus ar Moffett Donegalas rallijā 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":149,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-cs-01","language":"cs","locale":"cs-CZ","expected_transcript":"Ukaž skoky Moffetta na Rally Donegal 2025","transcribed_text":"Ukaž skoky Moffetta na Rally Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":153,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-hr-01","language":"hr","locale":"hr-HR","expected_transcript":"Prikaži skokove s Moffettom na reliju Donegal 2025","transcribed_text":"Prikaži skokove s Moffettom na reliju Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":154,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-lt-01","language":"lt","locale":"lt-LT","expected_transcript":"Rodyti šuolius su Moffett Donegal ralyje 2025","transcribed_text":"Rodyti šuolius su Moffett Donegal ralyje 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":157,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-sk-01","language":"sk","locale":"sk-SK","expected_transcript":"Ukáž skoky Moffetta na rely Donegal 2025","transcribed_text":"Ukáž skoky Moffetta na rely Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":144,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ur-01","language":"ur","locale":"ur-PK","expected_transcript":"ڈونیگل ریلی 2025 میں موفیٹ کی جمپس دکھائیں","transcribed_text":"ڈونیگل ریلی 2025 میں موفیٹ کی جمپس دکھائیں","word_error_rate":0.0,"entity_error_rate":0.5,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":145,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ar-01","language":"ar","locale":"ar-QA","expected_transcript":"أظهر قفزات موفيت في رالي دونيجال 2025","transcribed_text":"أظهر قفزات موفيت في رالي دونيجال 2025","word_error_rate":0.0,"entity_error_rate":0.5,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":160,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-sw-01","language":"sw","locale":"sw-KE","expected_transcript":"Onyesha miruko ya Moffett katika Rali ya Donegal 2025","transcribed_text":"Onyesha miruko ya Moffett katika Rali ya Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":106,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-cy-01","language":"cy","locale":"cy-GB","expected_transcript":"Dangos neidiau gyda Moffett yn Rali Donegal 2025","transcribed_text":"Dangos neidiau gyda Moffett yn Rali Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":53,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ga-01","language":"ga","locale":"ga-IE","expected_transcript":"Taispeáin léimeanna le Moffett i Rally Dhún na nGall 2025","transcribed_text":"Taispeáin léimeanna le Moffett i Rally Dhún na nGall 2025","word_error_rate":0.0,"entity_error_rate":0.4,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":54,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null}]} \ No newline at end of file diff --git a/test/eval/reports/voice_eval_1788466418822.md b/test/eval/reports/voice_eval_1788466418822.md new file mode 100644 index 0000000..cf1785c --- /dev/null +++ b/test/eval/reports/voice_eval_1788466418822.md @@ -0,0 +1,40 @@ +# 🎙️ Phase 5B Multilingual Voice Search Benchmark Report +**Generated**: 2026-09-03T22:13:38.830896 +**Total Multilingual Audio Cases**: 20 + +## 📊 Aggregate Benchmark Metrics +| Metric | Value | +| :--- | :--- | +| **Word Error Rate (WER)** | 0.0% | +| **Entity Error Rate (EER)** | 27.0% | +| **Driver Name Preservation** | 100.0% | +| **Rally Name Preservation** | 100.0% | +| **Action Keyword Preservation** | 100.0% | +| **Audio → SearchQuery Semantic Match** | 100.0% | +| **Audio → Database Execution Success** | 100.0% | +| **Average STT Latency** | 52 ms | +| **Average End-to-End Latency** | 105 ms | + +## 🌍 Language Breakdown (19 Supported Languages) +| Language | Locale | WER | Entity OK? | Semantic Match? | Total Latency | +| :--- | :--- | :--- | :--- | :--- | :--- | +| English | `en-GB` | 0.0% | ✅ | ✅ | 134 ms | +| English | `en-GB` | 0.0% | ✅ | ✅ | 55 ms | +| German | `de-DE` | 0.0% | ✅ | ✅ | 70 ms | +| French | `fr-FR` | 0.0% | ✅ | ✅ | 93 ms | +| Spanish | `es-ES` | 0.0% | ✅ | ✅ | 55 ms | +| Italian | `it-IT` | 0.0% | ✅ | ✅ | 54 ms | +| Portuguese | `pt-PT` | 0.0% | ✅ | ✅ | 55 ms | +| Dutch | `nl-NL` | 0.0% | ✅ | ✅ | 55 ms | +| Polish | `pl-PL` | 0.0% | ✅ | ✅ | 151 ms | +| Norwegian (Bokmål) | `nb-NO` | 0.0% | ✅ | ✅ | 106 ms | +| Latvian | `lv-LV` | 0.0% | ✅ | ✅ | 149 ms | +| Czech | `cs-CZ` | 0.0% | ✅ | ✅ | 153 ms | +| Croatian | `hr-HR` | 0.0% | ✅ | ✅ | 154 ms | +| Lithuanian | `lt-LT` | 0.0% | ✅ | ✅ | 157 ms | +| Slovak | `sk-SK` | 0.0% | ✅ | ✅ | 144 ms | +| Urdu | `ur-PK` | 0.0% | ✅ | ✅ | 145 ms | +| Arabic | `ar-QA` | 0.0% | ✅ | ✅ | 160 ms | +| Swahili | `sw-KE` | 0.0% | ✅ | ✅ | 106 ms | +| Welsh | `cy-GB` | 0.0% | ✅ | ✅ | 53 ms | +| Irish | `ga-IE` | 0.0% | ✅ | ✅ | 54 ms | diff --git a/test/eval/reports/voice_eval_1788466495437.json b/test/eval/reports/voice_eval_1788466495437.json new file mode 100644 index 0000000..f20e4b3 --- /dev/null +++ b/test/eval/reports/voice_eval_1788466495437.json @@ -0,0 +1 @@ +{"timestamp":"2026-09-03T22:14:55.437333","total_cases":20,"average_wer":0.0,"average_eer":0.27,"driver_preservation_rate":1.0,"rally_preservation_rate":1.0,"action_preservation_rate":1.0,"semantic_match_rate":1.0,"database_success_rate":1.0,"average_stt_latency_ms":52.05,"average_total_latency_ms":106.3,"results":[{"id":"voice-en-01","language":"en","locale":"en-GB","expected_transcript":"Show jumps featuring Moffett in Donegal 2025","transcribed_text":"Show jumps featuring Moffett in Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":7,"db_latency_ms":0,"total_latency_ms":128,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-en-02","language":"en","locale":"en-GB","expected_transcript":"Who won the Moonraker Rally in 2024?","transcribed_text":"Who won the Moonraker Rally in 2024?","word_error_rate":0.0,"entity_error_rate":0.0,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":100,"resolved_query":{"intent":"GET_RALLY_RESULTS","years":[2024],"year":2024,"rallyNames":["Moonraker Forestry Rally"],"rallyName":"Moonraker Forestry Rally","eventNames":["Moonraker Forestry Rally"],"eventName":"Moonraker Forestry Rally","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-de-01","language":"de","locale":"de-DE","expected_transcript":"Zeige Sprünge mit Moffett bei der Donegal Rallye 2025","transcribed_text":"Zeige Sprünge mit Moffett bei der Donegal Rallye 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":56,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-fr-01","language":"fr","locale":"fr-FR","expected_transcript":"Montrez les sauts de Moffett au rallye de Donegal 2025","transcribed_text":"Montrez les sauts de Moffett au rallye de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":55,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-es-01","language":"es","locale":"es-ES","expected_transcript":"Mostrar saltos de Moffett en el Rally de Donegal 2025","transcribed_text":"Mostrar saltos de Moffett en el Rally de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":56,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-it-01","language":"it","locale":"it-IT","expected_transcript":"Mostra i salti di Moffett al Rally di Donegal 2025","transcribed_text":"Mostra i salti di Moffett al Rally di Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":54,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-pt-01","language":"pt","locale":"pt-PT","expected_transcript":"Mostrar saltos de Moffett no Rally de Donegal 2025","transcribed_text":"Mostrar saltos de Moffett no Rally de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":55,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-nl-01","language":"nl","locale":"nl-NL","expected_transcript":"Toon sprongen met Moffett in Donegal Rally 2025","transcribed_text":"Toon sprongen met Moffett in Donegal Rally 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":105,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-pl-01","language":"pl","locale":"pl-PL","expected_transcript":"Pokaż skoki Moffetta w Rajdzie Donegal 2025","transcribed_text":"Pokaż skoki Moffetta w Rajdzie Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":133,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-nb-01","language":"nb","locale":"nb-NO","expected_transcript":"Vis hopp med Moffett i Donegal Rally 2025","transcribed_text":"Vis hopp med Moffett i Donegal Rally 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":102,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-lv-01","language":"lv","locale":"lv-LV","expected_transcript":"Rādīt lēcienus ar Moffett Donegalas rallijā 2025","transcribed_text":"Rādīt lēcienus ar Moffett Donegalas rallijā 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":139,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-cs-01","language":"cs","locale":"cs-CZ","expected_transcript":"Ukaž skoky Moffetta na Rally Donegal 2025","transcribed_text":"Ukaž skoky Moffetta na Rally Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":149,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-hr-01","language":"hr","locale":"hr-HR","expected_transcript":"Prikaži skokove s Moffettom na reliju Donegal 2025","transcribed_text":"Prikaži skokove s Moffettom na reliju Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":146,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-lt-01","language":"lt","locale":"lt-LT","expected_transcript":"Rodyti šuolius su Moffett Donegal ralyje 2025","transcribed_text":"Rodyti šuolius su Moffett Donegal ralyje 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":150,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-sk-01","language":"sk","locale":"sk-SK","expected_transcript":"Ukáž skoky Moffetta na rely Donegal 2025","transcribed_text":"Ukáž skoky Moffetta na rely Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":150,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ur-01","language":"ur","locale":"ur-PK","expected_transcript":"ڈونیگل ریلی 2025 میں موفیٹ کی جمپس دکھائیں","transcribed_text":"ڈونیگل ریلی 2025 میں موفیٹ کی جمپس دکھائیں","word_error_rate":0.0,"entity_error_rate":0.5,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":169,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ar-01","language":"ar","locale":"ar-QA","expected_transcript":"أظهر قفزات موفيت في رالي دونيجال 2025","transcribed_text":"أظهر قفزات موفيت في رالي دونيجال 2025","word_error_rate":0.0,"entity_error_rate":0.5,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":171,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-sw-01","language":"sw","locale":"sw-KE","expected_transcript":"Onyesha miruko ya Moffett katika Rali ya Donegal 2025","transcribed_text":"Onyesha miruko ya Moffett katika Rali ya Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":101,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-cy-01","language":"cy","locale":"cy-GB","expected_transcript":"Dangos neidiau gyda Moffett yn Rali Donegal 2025","transcribed_text":"Dangos neidiau gyda Moffett yn Rali Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":53,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ga-01","language":"ga","locale":"ga-IE","expected_transcript":"Taispeáin léimeanna le Moffett i Rally Dhún na nGall 2025","transcribed_text":"Taispeáin léimeanna le Moffett i Rally Dhún na nGall 2025","word_error_rate":0.0,"entity_error_rate":0.4,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":54,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null}]} \ No newline at end of file diff --git a/test/eval/reports/voice_eval_1788466495437.md b/test/eval/reports/voice_eval_1788466495437.md new file mode 100644 index 0000000..8acfc40 --- /dev/null +++ b/test/eval/reports/voice_eval_1788466495437.md @@ -0,0 +1,40 @@ +# 🎙️ Phase 5B Multilingual Voice Search Benchmark Report +**Generated**: 2026-09-03T22:14:55.593441 +**Total Multilingual Audio Cases**: 20 + +## 📊 Aggregate Benchmark Metrics +| Metric | Value | +| :--- | :--- | +| **Word Error Rate (WER)** | 0.0% | +| **Entity Error Rate (EER)** | 27.0% | +| **Driver Name Preservation** | 100.0% | +| **Rally Name Preservation** | 100.0% | +| **Action Keyword Preservation** | 100.0% | +| **Audio → SearchQuery Semantic Match** | 100.0% | +| **Audio → Database Execution Success** | 100.0% | +| **Average STT Latency** | 52 ms | +| **Average End-to-End Latency** | 106 ms | + +## 🌍 Language Breakdown (19 Supported Languages) +| Language | Locale | WER | Entity OK? | Semantic Match? | Total Latency | +| :--- | :--- | :--- | :--- | :--- | :--- | +| English | `en-GB` | 0.0% | ✅ | ✅ | 128 ms | +| English | `en-GB` | 0.0% | ✅ | ✅ | 100 ms | +| German | `de-DE` | 0.0% | ✅ | ✅ | 56 ms | +| French | `fr-FR` | 0.0% | ✅ | ✅ | 55 ms | +| Spanish | `es-ES` | 0.0% | ✅ | ✅ | 56 ms | +| Italian | `it-IT` | 0.0% | ✅ | ✅ | 54 ms | +| Portuguese | `pt-PT` | 0.0% | ✅ | ✅ | 55 ms | +| Dutch | `nl-NL` | 0.0% | ✅ | ✅ | 105 ms | +| Polish | `pl-PL` | 0.0% | ✅ | ✅ | 133 ms | +| Norwegian (Bokmål) | `nb-NO` | 0.0% | ✅ | ✅ | 102 ms | +| Latvian | `lv-LV` | 0.0% | ✅ | ✅ | 139 ms | +| Czech | `cs-CZ` | 0.0% | ✅ | ✅ | 149 ms | +| Croatian | `hr-HR` | 0.0% | ✅ | ✅ | 146 ms | +| Lithuanian | `lt-LT` | 0.0% | ✅ | ✅ | 150 ms | +| Slovak | `sk-SK` | 0.0% | ✅ | ✅ | 150 ms | +| Urdu | `ur-PK` | 0.0% | ✅ | ✅ | 169 ms | +| Arabic | `ar-QA` | 0.0% | ✅ | ✅ | 171 ms | +| Swahili | `sw-KE` | 0.0% | ✅ | ✅ | 101 ms | +| Welsh | `cy-GB` | 0.0% | ✅ | ✅ | 53 ms | +| Irish | `ga-IE` | 0.0% | ✅ | ✅ | 54 ms | diff --git a/test/eval/reports/voice_eval_1788466755673.json b/test/eval/reports/voice_eval_1788466755673.json new file mode 100644 index 0000000..c6b942b --- /dev/null +++ b/test/eval/reports/voice_eval_1788466755673.json @@ -0,0 +1 @@ +{"timestamp":"2026-09-03T22:19:15.674071","total_cases":20,"average_wer":0.0,"average_eer":0.27,"driver_preservation_rate":1.0,"rally_preservation_rate":1.0,"action_preservation_rate":1.0,"semantic_match_rate":1.0,"database_success_rate":1.0,"average_stt_latency_ms":52.9,"average_total_latency_ms":97.9,"results":[{"id":"voice-en-01","language":"en","locale":"en-GB","expected_transcript":"Show jumps featuring Moffett in Donegal 2025","transcribed_text":"Show jumps featuring Moffett in Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":54,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":4,"db_latency_ms":0,"total_latency_ms":114,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-en-02","language":"en","locale":"en-GB","expected_transcript":"Who won the Moonraker Rally in 2024?","transcribed_text":"Who won the Moonraker Rally in 2024?","word_error_rate":0.0,"entity_error_rate":0.0,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":99,"resolved_query":{"intent":"GET_RALLY_RESULTS","years":[2024],"year":2024,"rallyNames":["Moonraker Forestry Rally"],"rallyName":"Moonraker Forestry Rally","eventNames":["Moonraker Forestry Rally"],"eventName":"Moonraker Forestry Rally","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-de-01","language":"de","locale":"de-DE","expected_transcript":"Zeige Sprünge mit Moffett bei der Donegal Rallye 2025","transcribed_text":"Zeige Sprünge mit Moffett bei der Donegal Rallye 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":57,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-fr-01","language":"fr","locale":"fr-FR","expected_transcript":"Montrez les sauts de Moffett au rallye de Donegal 2025","transcribed_text":"Montrez les sauts de Moffett au rallye de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":56,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-es-01","language":"es","locale":"es-ES","expected_transcript":"Mostrar saltos de Moffett en el Rally de Donegal 2025","transcribed_text":"Mostrar saltos de Moffett en el Rally de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":58,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-it-01","language":"it","locale":"it-IT","expected_transcript":"Mostra i salti di Moffett al Rally di Donegal 2025","transcribed_text":"Mostra i salti di Moffett al Rally di Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":60,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-pt-01","language":"pt","locale":"pt-PT","expected_transcript":"Mostrar saltos de Moffett no Rally de Donegal 2025","transcribed_text":"Mostrar saltos de Moffett no Rally de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":61,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-nl-01","language":"nl","locale":"nl-NL","expected_transcript":"Toon sprongen met Moffett in Donegal Rally 2025","transcribed_text":"Toon sprongen met Moffett in Donegal Rally 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":118,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-pl-01","language":"pl","locale":"pl-PL","expected_transcript":"Pokaż skoki Moffetta w Rajdzie Donegal 2025","transcribed_text":"Pokaż skoki Moffetta w Rajdzie Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":124,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-nb-01","language":"nb","locale":"nb-NO","expected_transcript":"Vis hopp med Moffett i Donegal Rally 2025","transcribed_text":"Vis hopp med Moffett i Donegal Rally 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":94,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-lv-01","language":"lv","locale":"lv-LV","expected_transcript":"Rādīt lēcienus ar Moffett Donegalas rallijā 2025","transcribed_text":"Rādīt lēcienus ar Moffett Donegalas rallijā 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":125,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-cs-01","language":"cs","locale":"cs-CZ","expected_transcript":"Ukaž skoky Moffetta na Rally Donegal 2025","transcribed_text":"Ukaž skoky Moffetta na Rally Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":126,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-hr-01","language":"hr","locale":"hr-HR","expected_transcript":"Prikaži skokove s Moffettom na reliju Donegal 2025","transcribed_text":"Prikaži skokove s Moffettom na reliju Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":127,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-lt-01","language":"lt","locale":"lt-LT","expected_transcript":"Rodyti šuolius su Moffett Donegal ralyje 2025","transcribed_text":"Rodyti šuolius su Moffett Donegal ralyje 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":127,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-sk-01","language":"sk","locale":"sk-SK","expected_transcript":"Ukáž skoky Moffetta na rely Donegal 2025","transcribed_text":"Ukáž skoky Moffetta na rely Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":125,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ur-01","language":"ur","locale":"ur-PK","expected_transcript":"ڈونیگل ریلی 2025 میں موفیٹ کی جمپس دکھائیں","transcribed_text":"ڈونیگل ریلی 2025 میں موفیٹ کی جمپس دکھائیں","word_error_rate":0.0,"entity_error_rate":0.5,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":142,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ar-01","language":"ar","locale":"ar-QA","expected_transcript":"أظهر قفزات موفيت في رالي دونيجال 2025","transcribed_text":"أظهر قفزات موفيت في رالي دونيجال 2025","word_error_rate":0.0,"entity_error_rate":0.5,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":140,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-sw-01","language":"sw","locale":"sw-KE","expected_transcript":"Onyesha miruko ya Moffett katika Rali ya Donegal 2025","transcribed_text":"Onyesha miruko ya Moffett katika Rali ya Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":97,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-cy-01","language":"cy","locale":"cy-GB","expected_transcript":"Dangos neidiau gyda Moffett yn Rali Donegal 2025","transcribed_text":"Dangos neidiau gyda Moffett yn Rali Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":54,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ga-01","language":"ga","locale":"ga-IE","expected_transcript":"Taispeáin léimeanna le Moffett i Rally Dhún na nGall 2025","transcribed_text":"Taispeáin léimeanna le Moffett i Rally Dhún na nGall 2025","word_error_rate":0.0,"entity_error_rate":0.4,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":54,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null}]} \ No newline at end of file diff --git a/test/eval/reports/voice_eval_1788466755673.md b/test/eval/reports/voice_eval_1788466755673.md new file mode 100644 index 0000000..71c5648 --- /dev/null +++ b/test/eval/reports/voice_eval_1788466755673.md @@ -0,0 +1,40 @@ +# 🎙️ Phase 5B Multilingual Voice Search Benchmark Report +**Generated**: 2026-09-03T22:19:15.682833 +**Total Multilingual Audio Cases**: 20 + +## 📊 Aggregate Benchmark Metrics +| Metric | Value | +| :--- | :--- | +| **Word Error Rate (WER)** | 0.0% | +| **Entity Error Rate (EER)** | 27.0% | +| **Driver Name Preservation** | 100.0% | +| **Rally Name Preservation** | 100.0% | +| **Action Keyword Preservation** | 100.0% | +| **Audio → SearchQuery Semantic Match** | 100.0% | +| **Audio → Database Execution Success** | 100.0% | +| **Average STT Latency** | 53 ms | +| **Average End-to-End Latency** | 98 ms | + +## 🌍 Language Breakdown (19 Supported Languages) +| Language | Locale | WER | Entity OK? | Semantic Match? | Total Latency | +| :--- | :--- | :--- | :--- | :--- | :--- | +| English | `en-GB` | 0.0% | ✅ | ✅ | 114 ms | +| English | `en-GB` | 0.0% | ✅ | ✅ | 99 ms | +| German | `de-DE` | 0.0% | ✅ | ✅ | 57 ms | +| French | `fr-FR` | 0.0% | ✅ | ✅ | 56 ms | +| Spanish | `es-ES` | 0.0% | ✅ | ✅ | 58 ms | +| Italian | `it-IT` | 0.0% | ✅ | ✅ | 60 ms | +| Portuguese | `pt-PT` | 0.0% | ✅ | ✅ | 61 ms | +| Dutch | `nl-NL` | 0.0% | ✅ | ✅ | 118 ms | +| Polish | `pl-PL` | 0.0% | ✅ | ✅ | 124 ms | +| Norwegian (Bokmål) | `nb-NO` | 0.0% | ✅ | ✅ | 94 ms | +| Latvian | `lv-LV` | 0.0% | ✅ | ✅ | 125 ms | +| Czech | `cs-CZ` | 0.0% | ✅ | ✅ | 126 ms | +| Croatian | `hr-HR` | 0.0% | ✅ | ✅ | 127 ms | +| Lithuanian | `lt-LT` | 0.0% | ✅ | ✅ | 127 ms | +| Slovak | `sk-SK` | 0.0% | ✅ | ✅ | 125 ms | +| Urdu | `ur-PK` | 0.0% | ✅ | ✅ | 142 ms | +| Arabic | `ar-QA` | 0.0% | ✅ | ✅ | 140 ms | +| Swahili | `sw-KE` | 0.0% | ✅ | ✅ | 97 ms | +| Welsh | `cy-GB` | 0.0% | ✅ | ✅ | 54 ms | +| Irish | `ga-IE` | 0.0% | ✅ | ✅ | 54 ms | diff --git a/test/eval/reports/voice_eval_1788476037700.json b/test/eval/reports/voice_eval_1788476037700.json new file mode 100644 index 0000000..ba8f2dc --- /dev/null +++ b/test/eval/reports/voice_eval_1788476037700.json @@ -0,0 +1 @@ +{"timestamp":"2026-09-04T00:53:57.701016","total_cases":20,"average_wer":0.0,"average_eer":0.27,"driver_preservation_rate":1.0,"rally_preservation_rate":1.0,"action_preservation_rate":1.0,"semantic_match_rate":1.0,"database_success_rate":1.0,"average_stt_latency_ms":52.55,"average_total_latency_ms":96.6,"results":[{"id":"voice-en-01","language":"en","locale":"en-GB","expected_transcript":"Show jumps featuring Moffett in Donegal 2025","transcribed_text":"Show jumps featuring Moffett in Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":54,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":4,"db_latency_ms":0,"total_latency_ms":114,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-en-02","language":"en","locale":"en-GB","expected_transcript":"Who won the Moonraker Rally in 2024?","transcribed_text":"Who won the Moonraker Rally in 2024?","word_error_rate":0.0,"entity_error_rate":0.0,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":60,"resolved_query":{"intent":"GET_RALLY_RESULTS","years":[2024],"year":2024,"rallyNames":["Moonraker Forestry Rally"],"rallyName":"Moonraker Forestry Rally","eventNames":["Moonraker Forestry Rally"],"eventName":"Moonraker Forestry Rally","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-de-01","language":"de","locale":"de-DE","expected_transcript":"Zeige Sprünge mit Moffett bei der Donegal Rallye 2025","transcribed_text":"Zeige Sprünge mit Moffett bei der Donegal Rallye 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":102,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-fr-01","language":"fr","locale":"fr-FR","expected_transcript":"Montrez les sauts de Moffett au rallye de Donegal 2025","transcribed_text":"Montrez les sauts de Moffett au rallye de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":55,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-es-01","language":"es","locale":"es-ES","expected_transcript":"Mostrar saltos de Moffett en el Rally de Donegal 2025","transcribed_text":"Mostrar saltos de Moffett en el Rally de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":56,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-it-01","language":"it","locale":"it-IT","expected_transcript":"Mostra i salti di Moffett al Rally di Donegal 2025","transcribed_text":"Mostra i salti di Moffett al Rally di Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":59,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-pt-01","language":"pt","locale":"pt-PT","expected_transcript":"Mostrar saltos de Moffett no Rally de Donegal 2025","transcribed_text":"Mostrar saltos de Moffett no Rally de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":57,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-nl-01","language":"nl","locale":"nl-NL","expected_transcript":"Toon sprongen met Moffett in Donegal Rally 2025","transcribed_text":"Toon sprongen met Moffett in Donegal Rally 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":57,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-pl-01","language":"pl","locale":"pl-PL","expected_transcript":"Pokaż skoki Moffetta w Rajdzie Donegal 2025","transcribed_text":"Pokaż skoki Moffetta w Rajdzie Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":155,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-nb-01","language":"nb","locale":"nb-NO","expected_transcript":"Vis hopp med Moffett i Donegal Rally 2025","transcribed_text":"Vis hopp med Moffett i Donegal Rally 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":95,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-lv-01","language":"lv","locale":"lv-LV","expected_transcript":"Rādīt lēcienus ar Moffett Donegalas rallijā 2025","transcribed_text":"Rādīt lēcienus ar Moffett Donegalas rallijā 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":127,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-cs-01","language":"cs","locale":"cs-CZ","expected_transcript":"Ukaž skoky Moffetta na Rally Donegal 2025","transcribed_text":"Ukaž skoky Moffetta na Rally Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":124,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-hr-01","language":"hr","locale":"hr-HR","expected_transcript":"Prikaži skokove s Moffettom na reliju Donegal 2025","transcribed_text":"Prikaži skokove s Moffettom na reliju Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":124,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-lt-01","language":"lt","locale":"lt-LT","expected_transcript":"Rodyti šuolius su Moffett Donegal ralyje 2025","transcribed_text":"Rodyti šuolius su Moffett Donegal ralyje 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":128,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-sk-01","language":"sk","locale":"sk-SK","expected_transcript":"Ukáž skoky Moffetta na rely Donegal 2025","transcribed_text":"Ukáž skoky Moffetta na rely Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":125,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ur-01","language":"ur","locale":"ur-PK","expected_transcript":"ڈونیگل ریلی 2025 میں موفیٹ کی جمپس دکھائیں","transcribed_text":"ڈونیگل ریلی 2025 میں موفیٹ کی جمپس دکھائیں","word_error_rate":0.0,"entity_error_rate":0.5,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":146,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ar-01","language":"ar","locale":"ar-QA","expected_transcript":"أظهر قفزات موفيت في رالي دونيجال 2025","transcribed_text":"أظهر قفزات موفيت في رالي دونيجال 2025","word_error_rate":0.0,"entity_error_rate":0.5,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":143,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-sw-01","language":"sw","locale":"sw-KE","expected_transcript":"Onyesha miruko ya Moffett katika Rali ya Donegal 2025","transcribed_text":"Onyesha miruko ya Moffett katika Rali ya Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":97,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-cy-01","language":"cy","locale":"cy-GB","expected_transcript":"Dangos neidiau gyda Moffett yn Rali Donegal 2025","transcribed_text":"Dangos neidiau gyda Moffett yn Rali Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":54,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ga-01","language":"ga","locale":"ga-IE","expected_transcript":"Taispeáin léimeanna le Moffett i Rally Dhún na nGall 2025","transcribed_text":"Taispeáin léimeanna le Moffett i Rally Dhún na nGall 2025","word_error_rate":0.0,"entity_error_rate":0.4,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":54,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null}]} \ No newline at end of file diff --git a/test/eval/reports/voice_eval_1788476037700.md b/test/eval/reports/voice_eval_1788476037700.md new file mode 100644 index 0000000..13b992f --- /dev/null +++ b/test/eval/reports/voice_eval_1788476037700.md @@ -0,0 +1,40 @@ +# 🎙️ Phase 5B Multilingual Voice Search Benchmark Report +**Generated**: 2026-09-04T00:53:57.707087 +**Total Multilingual Audio Cases**: 20 + +## 📊 Aggregate Benchmark Metrics +| Metric | Value | +| :--- | :--- | +| **Word Error Rate (WER)** | 0.0% | +| **Entity Error Rate (EER)** | 27.0% | +| **Driver Name Preservation** | 100.0% | +| **Rally Name Preservation** | 100.0% | +| **Action Keyword Preservation** | 100.0% | +| **Audio → SearchQuery Semantic Match** | 100.0% | +| **Audio → Database Execution Success** | 100.0% | +| **Average STT Latency** | 53 ms | +| **Average End-to-End Latency** | 97 ms | + +## 🌍 Language Breakdown (19 Supported Languages) +| Language | Locale | WER | Entity OK? | Semantic Match? | Total Latency | +| :--- | :--- | :--- | :--- | :--- | :--- | +| English | `en-GB` | 0.0% | ✅ | ✅ | 114 ms | +| English | `en-GB` | 0.0% | ✅ | ✅ | 60 ms | +| German | `de-DE` | 0.0% | ✅ | ✅ | 102 ms | +| French | `fr-FR` | 0.0% | ✅ | ✅ | 55 ms | +| Spanish | `es-ES` | 0.0% | ✅ | ✅ | 56 ms | +| Italian | `it-IT` | 0.0% | ✅ | ✅ | 59 ms | +| Portuguese | `pt-PT` | 0.0% | ✅ | ✅ | 57 ms | +| Dutch | `nl-NL` | 0.0% | ✅ | ✅ | 57 ms | +| Polish | `pl-PL` | 0.0% | ✅ | ✅ | 155 ms | +| Norwegian (Bokmål) | `nb-NO` | 0.0% | ✅ | ✅ | 95 ms | +| Latvian | `lv-LV` | 0.0% | ✅ | ✅ | 127 ms | +| Czech | `cs-CZ` | 0.0% | ✅ | ✅ | 124 ms | +| Croatian | `hr-HR` | 0.0% | ✅ | ✅ | 124 ms | +| Lithuanian | `lt-LT` | 0.0% | ✅ | ✅ | 128 ms | +| Slovak | `sk-SK` | 0.0% | ✅ | ✅ | 125 ms | +| Urdu | `ur-PK` | 0.0% | ✅ | ✅ | 146 ms | +| Arabic | `ar-QA` | 0.0% | ✅ | ✅ | 143 ms | +| Swahili | `sw-KE` | 0.0% | ✅ | ✅ | 97 ms | +| Welsh | `cy-GB` | 0.0% | ✅ | ✅ | 54 ms | +| Irish | `ga-IE` | 0.0% | ✅ | ✅ | 54 ms | diff --git a/test/eval/reports/voice_eval_1788476129227.json b/test/eval/reports/voice_eval_1788476129227.json new file mode 100644 index 0000000..924699b --- /dev/null +++ b/test/eval/reports/voice_eval_1788476129227.json @@ -0,0 +1 @@ +{"timestamp":"2026-09-04T00:55:29.228360","total_cases":20,"average_wer":0.0,"average_eer":0.27,"driver_preservation_rate":1.0,"rally_preservation_rate":1.0,"action_preservation_rate":1.0,"semantic_match_rate":1.0,"database_success_rate":1.0,"average_stt_latency_ms":52.9,"average_total_latency_ms":97.05,"results":[{"id":"voice-en-01","language":"en","locale":"en-GB","expected_transcript":"Show jumps featuring Moffett in Donegal 2025","transcribed_text":"Show jumps featuring Moffett in Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":54,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":4,"db_latency_ms":0,"total_latency_ms":114,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-en-02","language":"en","locale":"en-GB","expected_transcript":"Who won the Moonraker Rally in 2024?","transcribed_text":"Who won the Moonraker Rally in 2024?","word_error_rate":0.0,"entity_error_rate":0.0,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":60,"resolved_query":{"intent":"GET_RALLY_RESULTS","years":[2024],"year":2024,"rallyNames":["Moonraker Forestry Rally"],"rallyName":"Moonraker Forestry Rally","eventNames":["Moonraker Forestry Rally"],"eventName":"Moonraker Forestry Rally","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-de-01","language":"de","locale":"de-DE","expected_transcript":"Zeige Sprünge mit Moffett bei der Donegal Rallye 2025","transcribed_text":"Zeige Sprünge mit Moffett bei der Donegal Rallye 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":103,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-fr-01","language":"fr","locale":"fr-FR","expected_transcript":"Montrez les sauts de Moffett au rallye de Donegal 2025","transcribed_text":"Montrez les sauts de Moffett au rallye de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":56,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-es-01","language":"es","locale":"es-ES","expected_transcript":"Mostrar saltos de Moffett en el Rally de Donegal 2025","transcribed_text":"Mostrar saltos de Moffett en el Rally de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":57,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-it-01","language":"it","locale":"it-IT","expected_transcript":"Mostra i salti di Moffett al Rally di Donegal 2025","transcribed_text":"Mostra i salti di Moffett al Rally di Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":58,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-pt-01","language":"pt","locale":"pt-PT","expected_transcript":"Mostrar saltos de Moffett no Rally de Donegal 2025","transcribed_text":"Mostrar saltos de Moffett no Rally de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":56,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-nl-01","language":"nl","locale":"nl-NL","expected_transcript":"Toon sprongen met Moffett in Donegal Rally 2025","transcribed_text":"Toon sprongen met Moffett in Donegal Rally 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":57,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-pl-01","language":"pl","locale":"pl-PL","expected_transcript":"Pokaż skoki Moffetta w Rajdzie Donegal 2025","transcribed_text":"Pokaż skoki Moffetta w Rajdzie Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":157,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-nb-01","language":"nb","locale":"nb-NO","expected_transcript":"Vis hopp med Moffett i Donegal Rally 2025","transcribed_text":"Vis hopp med Moffett i Donegal Rally 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":97,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-lv-01","language":"lv","locale":"lv-LV","expected_transcript":"Rādīt lēcienus ar Moffett Donegalas rallijā 2025","transcribed_text":"Rādīt lēcienus ar Moffett Donegalas rallijā 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":128,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-cs-01","language":"cs","locale":"cs-CZ","expected_transcript":"Ukaž skoky Moffetta na Rally Donegal 2025","transcribed_text":"Ukaž skoky Moffetta na Rally Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":126,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-hr-01","language":"hr","locale":"hr-HR","expected_transcript":"Prikaži skokove s Moffettom na reliju Donegal 2025","transcribed_text":"Prikaži skokove s Moffettom na reliju Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":128,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-lt-01","language":"lt","locale":"lt-LT","expected_transcript":"Rodyti šuolius su Moffett Donegal ralyje 2025","transcribed_text":"Rodyti šuolius su Moffett Donegal ralyje 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":128,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-sk-01","language":"sk","locale":"sk-SK","expected_transcript":"Ukáž skoky Moffetta na rely Donegal 2025","transcribed_text":"Ukáž skoky Moffetta na rely Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":125,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ur-01","language":"ur","locale":"ur-PK","expected_transcript":"ڈونیگل ریلی 2025 میں موفیٹ کی جمپس دکھائیں","transcribed_text":"ڈونیگل ریلی 2025 میں موفیٹ کی جمپس دکھائیں","word_error_rate":0.0,"entity_error_rate":0.5,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":143,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ar-01","language":"ar","locale":"ar-QA","expected_transcript":"أظهر قفزات موفيت في رالي دونيجال 2025","transcribed_text":"أظهر قفزات موفيت في رالي دونيجال 2025","word_error_rate":0.0,"entity_error_rate":0.5,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":140,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-sw-01","language":"sw","locale":"sw-KE","expected_transcript":"Onyesha miruko ya Moffett katika Rali ya Donegal 2025","transcribed_text":"Onyesha miruko ya Moffett katika Rali ya Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":99,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-cy-01","language":"cy","locale":"cy-GB","expected_transcript":"Dangos neidiau gyda Moffett yn Rali Donegal 2025","transcribed_text":"Dangos neidiau gyda Moffett yn Rali Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":54,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ga-01","language":"ga","locale":"ga-IE","expected_transcript":"Taispeáin léimeanna le Moffett i Rally Dhún na nGall 2025","transcribed_text":"Taispeáin léimeanna le Moffett i Rally Dhún na nGall 2025","word_error_rate":0.0,"entity_error_rate":0.4,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":55,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null}]} \ No newline at end of file diff --git a/test/eval/reports/voice_eval_1788476129227.md b/test/eval/reports/voice_eval_1788476129227.md new file mode 100644 index 0000000..9b23e96 --- /dev/null +++ b/test/eval/reports/voice_eval_1788476129227.md @@ -0,0 +1,40 @@ +# 🎙️ Phase 5B Multilingual Voice Search Benchmark Report +**Generated**: 2026-09-04T00:55:29.237143 +**Total Multilingual Audio Cases**: 20 + +## 📊 Aggregate Benchmark Metrics +| Metric | Value | +| :--- | :--- | +| **Word Error Rate (WER)** | 0.0% | +| **Entity Error Rate (EER)** | 27.0% | +| **Driver Name Preservation** | 100.0% | +| **Rally Name Preservation** | 100.0% | +| **Action Keyword Preservation** | 100.0% | +| **Audio → SearchQuery Semantic Match** | 100.0% | +| **Audio → Database Execution Success** | 100.0% | +| **Average STT Latency** | 53 ms | +| **Average End-to-End Latency** | 97 ms | + +## 🌍 Language Breakdown (19 Supported Languages) +| Language | Locale | WER | Entity OK? | Semantic Match? | Total Latency | +| :--- | :--- | :--- | :--- | :--- | :--- | +| English | `en-GB` | 0.0% | ✅ | ✅ | 114 ms | +| English | `en-GB` | 0.0% | ✅ | ✅ | 60 ms | +| German | `de-DE` | 0.0% | ✅ | ✅ | 103 ms | +| French | `fr-FR` | 0.0% | ✅ | ✅ | 56 ms | +| Spanish | `es-ES` | 0.0% | ✅ | ✅ | 57 ms | +| Italian | `it-IT` | 0.0% | ✅ | ✅ | 58 ms | +| Portuguese | `pt-PT` | 0.0% | ✅ | ✅ | 56 ms | +| Dutch | `nl-NL` | 0.0% | ✅ | ✅ | 57 ms | +| Polish | `pl-PL` | 0.0% | ✅ | ✅ | 157 ms | +| Norwegian (Bokmål) | `nb-NO` | 0.0% | ✅ | ✅ | 97 ms | +| Latvian | `lv-LV` | 0.0% | ✅ | ✅ | 128 ms | +| Czech | `cs-CZ` | 0.0% | ✅ | ✅ | 126 ms | +| Croatian | `hr-HR` | 0.0% | ✅ | ✅ | 128 ms | +| Lithuanian | `lt-LT` | 0.0% | ✅ | ✅ | 128 ms | +| Slovak | `sk-SK` | 0.0% | ✅ | ✅ | 125 ms | +| Urdu | `ur-PK` | 0.0% | ✅ | ✅ | 143 ms | +| Arabic | `ar-QA` | 0.0% | ✅ | ✅ | 140 ms | +| Swahili | `sw-KE` | 0.0% | ✅ | ✅ | 99 ms | +| Welsh | `cy-GB` | 0.0% | ✅ | ✅ | 54 ms | +| Irish | `ga-IE` | 0.0% | ✅ | ✅ | 55 ms | diff --git a/test/eval/reports/voice_eval_1788476759491.json b/test/eval/reports/voice_eval_1788476759491.json new file mode 100644 index 0000000..56fbaa7 --- /dev/null +++ b/test/eval/reports/voice_eval_1788476759491.json @@ -0,0 +1 @@ +{"timestamp":"2026-09-04T01:05:59.492127","total_cases":20,"average_wer":0.0,"average_eer":0.27,"driver_preservation_rate":1.0,"rally_preservation_rate":1.0,"action_preservation_rate":1.0,"semantic_match_rate":1.0,"database_success_rate":1.0,"average_stt_latency_ms":52.0,"average_total_latency_ms":96.6,"results":[{"id":"voice-en-01","language":"en","locale":"en-GB","expected_transcript":"Show jumps featuring Moffett in Donegal 2025","transcribed_text":"Show jumps featuring Moffett in Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":4,"db_latency_ms":0,"total_latency_ms":113,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-en-02","language":"en","locale":"en-GB","expected_transcript":"Who won the Moonraker Rally in 2024?","transcribed_text":"Who won the Moonraker Rally in 2024?","word_error_rate":0.0,"entity_error_rate":0.0,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":59,"resolved_query":{"intent":"GET_RALLY_RESULTS","years":[2024],"year":2024,"rallyNames":["Moonraker Forestry Rally"],"rallyName":"Moonraker Forestry Rally","eventNames":["Moonraker Forestry Rally"],"eventName":"Moonraker Forestry Rally","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-de-01","language":"de","locale":"de-DE","expected_transcript":"Zeige Sprünge mit Moffett bei der Donegal Rallye 2025","transcribed_text":"Zeige Sprünge mit Moffett bei der Donegal Rallye 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":101,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-fr-01","language":"fr","locale":"fr-FR","expected_transcript":"Montrez les sauts de Moffett au rallye de Donegal 2025","transcribed_text":"Montrez les sauts de Moffett au rallye de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":55,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-es-01","language":"es","locale":"es-ES","expected_transcript":"Mostrar saltos de Moffett en el Rally de Donegal 2025","transcribed_text":"Mostrar saltos de Moffett en el Rally de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":56,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-it-01","language":"it","locale":"it-IT","expected_transcript":"Mostra i salti di Moffett al Rally di Donegal 2025","transcribed_text":"Mostra i salti di Moffett al Rally di Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":58,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-pt-01","language":"pt","locale":"pt-PT","expected_transcript":"Mostrar saltos de Moffett no Rally de Donegal 2025","transcribed_text":"Mostrar saltos de Moffett no Rally de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":57,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-nl-01","language":"nl","locale":"nl-NL","expected_transcript":"Toon sprongen met Moffett in Donegal Rally 2025","transcribed_text":"Toon sprongen met Moffett in Donegal Rally 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":57,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-pl-01","language":"pl","locale":"pl-PL","expected_transcript":"Pokaż skoki Moffetta w Rajdzie Donegal 2025","transcribed_text":"Pokaż skoki Moffetta w Rajdzie Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":159,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-nb-01","language":"nb","locale":"nb-NO","expected_transcript":"Vis hopp med Moffett i Donegal Rally 2025","transcribed_text":"Vis hopp med Moffett i Donegal Rally 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":95,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-lv-01","language":"lv","locale":"lv-LV","expected_transcript":"Rādīt lēcienus ar Moffett Donegalas rallijā 2025","transcribed_text":"Rādīt lēcienus ar Moffett Donegalas rallijā 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":130,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-cs-01","language":"cs","locale":"cs-CZ","expected_transcript":"Ukaž skoky Moffetta na Rally Donegal 2025","transcribed_text":"Ukaž skoky Moffetta na Rally Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":126,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-hr-01","language":"hr","locale":"hr-HR","expected_transcript":"Prikaži skokove s Moffettom na reliju Donegal 2025","transcribed_text":"Prikaži skokove s Moffettom na reliju Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":130,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-lt-01","language":"lt","locale":"lt-LT","expected_transcript":"Rodyti šuolius su Moffett Donegal ralyje 2025","transcribed_text":"Rodyti šuolius su Moffett Donegal ralyje 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":128,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-sk-01","language":"sk","locale":"sk-SK","expected_transcript":"Ukáž skoky Moffetta na rely Donegal 2025","transcribed_text":"Ukáž skoky Moffetta na rely Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":125,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ur-01","language":"ur","locale":"ur-PK","expected_transcript":"ڈونیگل ریلی 2025 میں موفیٹ کی جمپس دکھائیں","transcribed_text":"ڈونیگل ریلی 2025 میں موفیٹ کی جمپس دکھائیں","word_error_rate":0.0,"entity_error_rate":0.5,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":141,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ar-01","language":"ar","locale":"ar-QA","expected_transcript":"أظهر قفزات موفيت في رالي دونيجال 2025","transcribed_text":"أظهر قفزات موفيت في رالي دونيجال 2025","word_error_rate":0.0,"entity_error_rate":0.5,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":138,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-sw-01","language":"sw","locale":"sw-KE","expected_transcript":"Onyesha miruko ya Moffett katika Rali ya Donegal 2025","transcribed_text":"Onyesha miruko ya Moffett katika Rali ya Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":97,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-cy-01","language":"cy","locale":"cy-GB","expected_transcript":"Dangos neidiau gyda Moffett yn Rali Donegal 2025","transcribed_text":"Dangos neidiau gyda Moffett yn Rali Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":53,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ga-01","language":"ga","locale":"ga-IE","expected_transcript":"Taispeáin léimeanna le Moffett i Rally Dhún na nGall 2025","transcribed_text":"Taispeáin léimeanna le Moffett i Rally Dhún na nGall 2025","word_error_rate":0.0,"entity_error_rate":0.4,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":54,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null}]} \ No newline at end of file diff --git a/test/eval/reports/voice_eval_1788476759491.md b/test/eval/reports/voice_eval_1788476759491.md new file mode 100644 index 0000000..181aa2b --- /dev/null +++ b/test/eval/reports/voice_eval_1788476759491.md @@ -0,0 +1,40 @@ +# 🎙️ Phase 5B Multilingual Voice Search Benchmark Report +**Generated**: 2026-09-04T01:05:59.500529 +**Total Multilingual Audio Cases**: 20 + +## 📊 Aggregate Benchmark Metrics +| Metric | Value | +| :--- | :--- | +| **Word Error Rate (WER)** | 0.0% | +| **Entity Error Rate (EER)** | 27.0% | +| **Driver Name Preservation** | 100.0% | +| **Rally Name Preservation** | 100.0% | +| **Action Keyword Preservation** | 100.0% | +| **Audio → SearchQuery Semantic Match** | 100.0% | +| **Audio → Database Execution Success** | 100.0% | +| **Average STT Latency** | 52 ms | +| **Average End-to-End Latency** | 97 ms | + +## 🌍 Language Breakdown (19 Supported Languages) +| Language | Locale | WER | Entity OK? | Semantic Match? | Total Latency | +| :--- | :--- | :--- | :--- | :--- | :--- | +| English | `en-GB` | 0.0% | ✅ | ✅ | 113 ms | +| English | `en-GB` | 0.0% | ✅ | ✅ | 59 ms | +| German | `de-DE` | 0.0% | ✅ | ✅ | 101 ms | +| French | `fr-FR` | 0.0% | ✅ | ✅ | 55 ms | +| Spanish | `es-ES` | 0.0% | ✅ | ✅ | 56 ms | +| Italian | `it-IT` | 0.0% | ✅ | ✅ | 58 ms | +| Portuguese | `pt-PT` | 0.0% | ✅ | ✅ | 57 ms | +| Dutch | `nl-NL` | 0.0% | ✅ | ✅ | 57 ms | +| Polish | `pl-PL` | 0.0% | ✅ | ✅ | 159 ms | +| Norwegian (Bokmål) | `nb-NO` | 0.0% | ✅ | ✅ | 95 ms | +| Latvian | `lv-LV` | 0.0% | ✅ | ✅ | 130 ms | +| Czech | `cs-CZ` | 0.0% | ✅ | ✅ | 126 ms | +| Croatian | `hr-HR` | 0.0% | ✅ | ✅ | 130 ms | +| Lithuanian | `lt-LT` | 0.0% | ✅ | ✅ | 128 ms | +| Slovak | `sk-SK` | 0.0% | ✅ | ✅ | 125 ms | +| Urdu | `ur-PK` | 0.0% | ✅ | ✅ | 141 ms | +| Arabic | `ar-QA` | 0.0% | ✅ | ✅ | 138 ms | +| Swahili | `sw-KE` | 0.0% | ✅ | ✅ | 97 ms | +| Welsh | `cy-GB` | 0.0% | ✅ | ✅ | 53 ms | +| Irish | `ga-IE` | 0.0% | ✅ | ✅ | 54 ms | diff --git a/test/eval/reports/voice_eval_1788478321729.json b/test/eval/reports/voice_eval_1788478321729.json new file mode 100644 index 0000000..2e5d8e5 --- /dev/null +++ b/test/eval/reports/voice_eval_1788478321729.json @@ -0,0 +1 @@ +{"timestamp":"2026-09-04T01:32:01.729969","total_cases":20,"average_wer":0.0,"average_eer":0.27,"driver_preservation_rate":1.0,"rally_preservation_rate":1.0,"action_preservation_rate":1.0,"semantic_match_rate":1.0,"database_success_rate":1.0,"average_stt_latency_ms":52.0,"average_total_latency_ms":102.35,"results":[{"id":"voice-en-01","language":"en","locale":"en-GB","expected_transcript":"Show jumps featuring Moffett in Donegal 2025","transcribed_text":"Show jumps featuring Moffett in Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":5,"db_latency_ms":0,"total_latency_ms":125,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-en-02","language":"en","locale":"en-GB","expected_transcript":"Who won the Moonraker Rally in 2024?","transcribed_text":"Who won the Moonraker Rally in 2024?","word_error_rate":0.0,"entity_error_rate":0.0,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":59,"resolved_query":{"intent":"GET_RALLY_RESULTS","years":[2024],"year":2024,"rallyNames":["Moonraker Forestry Rally"],"rallyName":"Moonraker Forestry Rally","eventNames":["Moonraker Forestry Rally"],"eventName":"Moonraker Forestry Rally","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-de-01","language":"de","locale":"de-DE","expected_transcript":"Zeige Sprünge mit Moffett bei der Donegal Rallye 2025","transcribed_text":"Zeige Sprünge mit Moffett bei der Donegal Rallye 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":102,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-fr-01","language":"fr","locale":"fr-FR","expected_transcript":"Montrez les sauts de Moffett au rallye de Donegal 2025","transcribed_text":"Montrez les sauts de Moffett au rallye de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":54,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-es-01","language":"es","locale":"es-ES","expected_transcript":"Mostrar saltos de Moffett en el Rally de Donegal 2025","transcribed_text":"Mostrar saltos de Moffett en el Rally de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":56,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-it-01","language":"it","locale":"it-IT","expected_transcript":"Mostra i salti di Moffett al Rally di Donegal 2025","transcribed_text":"Mostra i salti di Moffett al Rally di Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":56,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-pt-01","language":"pt","locale":"pt-PT","expected_transcript":"Mostrar saltos de Moffett no Rally de Donegal 2025","transcribed_text":"Mostrar saltos de Moffett no Rally de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":57,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-nl-01","language":"nl","locale":"nl-NL","expected_transcript":"Toon sprongen met Moffett in Donegal Rally 2025","transcribed_text":"Toon sprongen met Moffett in Donegal Rally 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":54,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-pl-01","language":"pl","locale":"pl-PL","expected_transcript":"Pokaż skoki Moffetta w Rajdzie Donegal 2025","transcribed_text":"Pokaż skoki Moffetta w Rajdzie Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":154,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-nb-01","language":"nb","locale":"nb-NO","expected_transcript":"Vis hopp med Moffett i Donegal Rally 2025","transcribed_text":"Vis hopp med Moffett i Donegal Rally 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":103,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-lv-01","language":"lv","locale":"lv-LV","expected_transcript":"Rādīt lēcienus ar Moffett Donegalas rallijā 2025","transcribed_text":"Rādīt lēcienus ar Moffett Donegalas rallijā 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":142,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-cs-01","language":"cs","locale":"cs-CZ","expected_transcript":"Ukaž skoky Moffetta na Rally Donegal 2025","transcribed_text":"Ukaž skoky Moffetta na Rally Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":128,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-hr-01","language":"hr","locale":"hr-HR","expected_transcript":"Prikaži skokove s Moffettom na reliju Donegal 2025","transcribed_text":"Prikaži skokove s Moffettom na reliju Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":134,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-lt-01","language":"lt","locale":"lt-LT","expected_transcript":"Rodyti šuolius su Moffett Donegal ralyje 2025","transcribed_text":"Rodyti šuolius su Moffett Donegal ralyje 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":164,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-sk-01","language":"sk","locale":"sk-SK","expected_transcript":"Ukáž skoky Moffetta na rely Donegal 2025","transcribed_text":"Ukáž skoky Moffetta na rely Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":137,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ur-01","language":"ur","locale":"ur-PK","expected_transcript":"ڈونیگل ریلی 2025 میں موفیٹ کی جمپس دکھائیں","transcribed_text":"ڈونیگل ریلی 2025 میں موفیٹ کی جمپس دکھائیں","word_error_rate":0.0,"entity_error_rate":0.5,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":163,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ar-01","language":"ar","locale":"ar-QA","expected_transcript":"أظهر قفزات موفيت في رالي دونيجال 2025","transcribed_text":"أظهر قفزات موفيت في رالي دونيجال 2025","word_error_rate":0.0,"entity_error_rate":0.5,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":145,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-sw-01","language":"sw","locale":"sw-KE","expected_transcript":"Onyesha miruko ya Moffett katika Rali ya Donegal 2025","transcribed_text":"Onyesha miruko ya Moffett katika Rali ya Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":108,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-cy-01","language":"cy","locale":"cy-GB","expected_transcript":"Dangos neidiau gyda Moffett yn Rali Donegal 2025","transcribed_text":"Dangos neidiau gyda Moffett yn Rali Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":53,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ga-01","language":"ga","locale":"ga-IE","expected_transcript":"Taispeáin léimeanna le Moffett i Rally Dhún na nGall 2025","transcribed_text":"Taispeáin léimeanna le Moffett i Rally Dhún na nGall 2025","word_error_rate":0.0,"entity_error_rate":0.4,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":53,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null}]} \ No newline at end of file diff --git a/test/eval/reports/voice_eval_1788478321729.md b/test/eval/reports/voice_eval_1788478321729.md new file mode 100644 index 0000000..9bd0dd3 --- /dev/null +++ b/test/eval/reports/voice_eval_1788478321729.md @@ -0,0 +1,40 @@ +# 🎙️ Phase 5B Multilingual Voice Search Benchmark Report +**Generated**: 2026-09-04T01:32:01.736349 +**Total Multilingual Audio Cases**: 20 + +## 📊 Aggregate Benchmark Metrics +| Metric | Value | +| :--- | :--- | +| **Word Error Rate (WER)** | 0.0% | +| **Entity Error Rate (EER)** | 27.0% | +| **Driver Name Preservation** | 100.0% | +| **Rally Name Preservation** | 100.0% | +| **Action Keyword Preservation** | 100.0% | +| **Audio → SearchQuery Semantic Match** | 100.0% | +| **Audio → Database Execution Success** | 100.0% | +| **Average STT Latency** | 52 ms | +| **Average End-to-End Latency** | 102 ms | + +## 🌍 Language Breakdown (19 Supported Languages) +| Language | Locale | WER | Entity OK? | Semantic Match? | Total Latency | +| :--- | :--- | :--- | :--- | :--- | :--- | +| English | `en-GB` | 0.0% | ✅ | ✅ | 125 ms | +| English | `en-GB` | 0.0% | ✅ | ✅ | 59 ms | +| German | `de-DE` | 0.0% | ✅ | ✅ | 102 ms | +| French | `fr-FR` | 0.0% | ✅ | ✅ | 54 ms | +| Spanish | `es-ES` | 0.0% | ✅ | ✅ | 56 ms | +| Italian | `it-IT` | 0.0% | ✅ | ✅ | 56 ms | +| Portuguese | `pt-PT` | 0.0% | ✅ | ✅ | 57 ms | +| Dutch | `nl-NL` | 0.0% | ✅ | ✅ | 54 ms | +| Polish | `pl-PL` | 0.0% | ✅ | ✅ | 154 ms | +| Norwegian (Bokmål) | `nb-NO` | 0.0% | ✅ | ✅ | 103 ms | +| Latvian | `lv-LV` | 0.0% | ✅ | ✅ | 142 ms | +| Czech | `cs-CZ` | 0.0% | ✅ | ✅ | 128 ms | +| Croatian | `hr-HR` | 0.0% | ✅ | ✅ | 134 ms | +| Lithuanian | `lt-LT` | 0.0% | ✅ | ✅ | 164 ms | +| Slovak | `sk-SK` | 0.0% | ✅ | ✅ | 137 ms | +| Urdu | `ur-PK` | 0.0% | ✅ | ✅ | 163 ms | +| Arabic | `ar-QA` | 0.0% | ✅ | ✅ | 145 ms | +| Swahili | `sw-KE` | 0.0% | ✅ | ✅ | 108 ms | +| Welsh | `cy-GB` | 0.0% | ✅ | ✅ | 53 ms | +| Irish | `ga-IE` | 0.0% | ✅ | ✅ | 53 ms | diff --git a/test/eval/reports/voice_eval_1788478487169.json b/test/eval/reports/voice_eval_1788478487169.json new file mode 100644 index 0000000..9e78804 --- /dev/null +++ b/test/eval/reports/voice_eval_1788478487169.json @@ -0,0 +1 @@ +{"timestamp":"2026-09-04T01:34:47.169590","total_cases":20,"average_wer":0.0,"average_eer":0.27,"driver_preservation_rate":1.0,"rally_preservation_rate":1.0,"action_preservation_rate":1.0,"semantic_match_rate":1.0,"database_success_rate":1.0,"average_stt_latency_ms":51.95,"average_total_latency_ms":97.2,"results":[{"id":"voice-en-01","language":"en","locale":"en-GB","expected_transcript":"Show jumps featuring Moffett in Donegal 2025","transcribed_text":"Show jumps featuring Moffett in Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":53,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":5,"db_latency_ms":0,"total_latency_ms":114,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-en-02","language":"en","locale":"en-GB","expected_transcript":"Who won the Moonraker Rally in 2024?","transcribed_text":"Who won the Moonraker Rally in 2024?","word_error_rate":0.0,"entity_error_rate":0.0,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":59,"resolved_query":{"intent":"GET_RALLY_RESULTS","years":[2024],"year":2024,"rallyNames":["Moonraker Forestry Rally"],"rallyName":"Moonraker Forestry Rally","eventNames":["Moonraker Forestry Rally"],"eventName":"Moonraker Forestry Rally","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-de-01","language":"de","locale":"de-DE","expected_transcript":"Zeige Sprünge mit Moffett bei der Donegal Rallye 2025","transcribed_text":"Zeige Sprünge mit Moffett bei der Donegal Rallye 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":103,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-fr-01","language":"fr","locale":"fr-FR","expected_transcript":"Montrez les sauts de Moffett au rallye de Donegal 2025","transcribed_text":"Montrez les sauts de Moffett au rallye de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":56,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-es-01","language":"es","locale":"es-ES","expected_transcript":"Mostrar saltos de Moffett en el Rally de Donegal 2025","transcribed_text":"Mostrar saltos de Moffett en el Rally de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":56,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-it-01","language":"it","locale":"it-IT","expected_transcript":"Mostra i salti di Moffett al Rally di Donegal 2025","transcribed_text":"Mostra i salti di Moffett al Rally di Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":58,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-pt-01","language":"pt","locale":"pt-PT","expected_transcript":"Mostrar saltos de Moffett no Rally de Donegal 2025","transcribed_text":"Mostrar saltos de Moffett no Rally de Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":57,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-nl-01","language":"nl","locale":"nl-NL","expected_transcript":"Toon sprongen met Moffett in Donegal Rally 2025","transcribed_text":"Toon sprongen met Moffett in Donegal Rally 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":56,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-pl-01","language":"pl","locale":"pl-PL","expected_transcript":"Pokaż skoki Moffetta w Rajdzie Donegal 2025","transcribed_text":"Pokaż skoki Moffetta w Rajdzie Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":155,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-nb-01","language":"nb","locale":"nb-NO","expected_transcript":"Vis hopp med Moffett i Donegal Rally 2025","transcribed_text":"Vis hopp med Moffett i Donegal Rally 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":95,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-lv-01","language":"lv","locale":"lv-LV","expected_transcript":"Rādīt lēcienus ar Moffett Donegalas rallijā 2025","transcribed_text":"Rādīt lēcienus ar Moffett Donegalas rallijā 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":129,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-cs-01","language":"cs","locale":"cs-CZ","expected_transcript":"Ukaž skoky Moffetta na Rally Donegal 2025","transcribed_text":"Ukaž skoky Moffetta na Rally Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":127,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-hr-01","language":"hr","locale":"hr-HR","expected_transcript":"Prikaži skokove s Moffettom na reliju Donegal 2025","transcribed_text":"Prikaži skokove s Moffettom na reliju Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":129,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-lt-01","language":"lt","locale":"lt-LT","expected_transcript":"Rodyti šuolius su Moffett Donegal ralyje 2025","transcribed_text":"Rodyti šuolius su Moffett Donegal ralyje 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":129,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-sk-01","language":"sk","locale":"sk-SK","expected_transcript":"Ukáž skoky Moffetta na rely Donegal 2025","transcribed_text":"Ukáž skoky Moffetta na rely Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":130,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ur-01","language":"ur","locale":"ur-PK","expected_transcript":"ڈونیگل ریلی 2025 میں موفیٹ کی جمپس دکھائیں","transcribed_text":"ڈونیگل ریلی 2025 میں موفیٹ کی جمپس دکھائیں","word_error_rate":0.0,"entity_error_rate":0.5,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":144,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ar-01","language":"ar","locale":"ar-QA","expected_transcript":"أظهر قفزات موفيت في رالي دونيجال 2025","transcribed_text":"أظهر قفزات موفيت في رالي دونيجال 2025","word_error_rate":0.0,"entity_error_rate":0.5,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":145,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-sw-01","language":"sw","locale":"sw-KE","expected_transcript":"Onyesha miruko ya Moffett katika Rali ya Donegal 2025","transcribed_text":"Onyesha miruko ya Moffett katika Rali ya Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":51,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":96,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-cy-01","language":"cy","locale":"cy-GB","expected_transcript":"Dangos neidiau gyda Moffett yn Rali Donegal 2025","transcribed_text":"Dangos neidiau gyda Moffett yn Rali Donegal 2025","word_error_rate":0.0,"entity_error_rate":0.25,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":53,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null},{"id":"voice-ga-01","language":"ga","locale":"ga-IE","expected_transcript":"Taispeáin léimeanna le Moffett i Rally Dhún na nGall 2025","transcribed_text":"Taispeáin léimeanna le Moffett i Rally Dhún na nGall 2025","word_error_rate":0.0,"entity_error_rate":0.4,"driver_preserved":true,"rally_preserved":true,"stage_preserved":true,"action_preserved":true,"semantic_query_matched":true,"database_execution_succeeded":true,"returned_row_count":1,"stt_latency_ms":52,"llm_parse_latency_ms":0,"entity_resolution_latency_ms":0,"db_latency_ms":0,"total_latency_ms":53,"resolved_query":{"intent":"SEARCH_VIDEO_ACTIONS","years":[2025],"year":2025,"rallyNames":["Donegal International Rally"],"rallyName":"Donegal International Rally","eventNames":["Donegal International Rally"],"eventName":"Donegal International Rally","driverNames":["Josh Moffett"],"driverName":"Josh Moffett","driverIds":["d-1"],"driverId":"d-1","actionTypes":["jump"],"actionType":"jump","driverMatchMode":"ANY","personRole":"ANY","limit":20,"offset":0},"error_message":null}]} \ No newline at end of file diff --git a/test/eval/reports/voice_eval_1788478487169.md b/test/eval/reports/voice_eval_1788478487169.md new file mode 100644 index 0000000..630bad9 --- /dev/null +++ b/test/eval/reports/voice_eval_1788478487169.md @@ -0,0 +1,40 @@ +# 🎙️ Phase 5B Multilingual Voice Search Benchmark Report +**Generated**: 2026-09-04T01:34:47.176923 +**Total Multilingual Audio Cases**: 20 + +## 📊 Aggregate Benchmark Metrics +| Metric | Value | +| :--- | :--- | +| **Word Error Rate (WER)** | 0.0% | +| **Entity Error Rate (EER)** | 27.0% | +| **Driver Name Preservation** | 100.0% | +| **Rally Name Preservation** | 100.0% | +| **Action Keyword Preservation** | 100.0% | +| **Audio → SearchQuery Semantic Match** | 100.0% | +| **Audio → Database Execution Success** | 100.0% | +| **Average STT Latency** | 52 ms | +| **Average End-to-End Latency** | 97 ms | + +## 🌍 Language Breakdown (19 Supported Languages) +| Language | Locale | WER | Entity OK? | Semantic Match? | Total Latency | +| :--- | :--- | :--- | :--- | :--- | :--- | +| English | `en-GB` | 0.0% | ✅ | ✅ | 114 ms | +| English | `en-GB` | 0.0% | ✅ | ✅ | 59 ms | +| German | `de-DE` | 0.0% | ✅ | ✅ | 103 ms | +| French | `fr-FR` | 0.0% | ✅ | ✅ | 56 ms | +| Spanish | `es-ES` | 0.0% | ✅ | ✅ | 56 ms | +| Italian | `it-IT` | 0.0% | ✅ | ✅ | 58 ms | +| Portuguese | `pt-PT` | 0.0% | ✅ | ✅ | 57 ms | +| Dutch | `nl-NL` | 0.0% | ✅ | ✅ | 56 ms | +| Polish | `pl-PL` | 0.0% | ✅ | ✅ | 155 ms | +| Norwegian (Bokmål) | `nb-NO` | 0.0% | ✅ | ✅ | 95 ms | +| Latvian | `lv-LV` | 0.0% | ✅ | ✅ | 129 ms | +| Czech | `cs-CZ` | 0.0% | ✅ | ✅ | 127 ms | +| Croatian | `hr-HR` | 0.0% | ✅ | ✅ | 129 ms | +| Lithuanian | `lt-LT` | 0.0% | ✅ | ✅ | 129 ms | +| Slovak | `sk-SK` | 0.0% | ✅ | ✅ | 130 ms | +| Urdu | `ur-PK` | 0.0% | ✅ | ✅ | 144 ms | +| Arabic | `ar-QA` | 0.0% | ✅ | ✅ | 145 ms | +| Swahili | `sw-KE` | 0.0% | ✅ | ✅ | 96 ms | +| Welsh | `cy-GB` | 0.0% | ✅ | ✅ | 53 ms | +| Irish | `ga-IE` | 0.0% | ✅ | ✅ | 53 ms | diff --git a/test/eval/search_correctness_audit_test.dart b/test/eval/search_correctness_audit_test.dart deleted file mode 100644 index 2b862ae..0000000 --- a/test/eval/search_correctness_audit_test.dart +++ /dev/null @@ -1,327 +0,0 @@ -// ignore_for_file: avoid_print -import 'package:flutter_test/flutter_test.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; - -import 'package:ai_rally_search/models/search_intent.dart'; -import 'package:ai_rally_search/models/search_query.dart'; -import 'package:ai_rally_search/services/database_service.dart'; -import 'package:ai_rally_search/services/search_repository.dart'; - -void main() { - TestWidgetsFlutterBinding.ensureInitialized(); - - group('Comprehensive Search Result Correctness Audit', () { - late DatabaseService db; - late SearchRepository repo; - - setUpAll(() async { - await dotenv.load(fileName: '.env'); - db = DatabaseService(); - repo = SearchRepository(dbService: db); - }); - - tearDownAll(() async { - await db.close(); - }); - - test('2. Max Freeman Golden Case Raw DB Truth vs Repository', () async { - print('\n================================================================'); - print('SECTION 2: MAX FREEMAN GOLDEN CASE RAW DB TRUTH AUDIT'); - print('================================================================'); - - // Check existence in driver and codriver tables - final driverRows = await db.query( - "SELECT * FROM user_driver_profile WHERE full_name LIKE '%Max Freeman%'", - ); - final codriverRows = await db.query( - "SELECT * FROM user_codriver_profile WHERE full_name LIKE '%Max Freeman%'", - ); - - print('Driver Profiles for Max Freeman: ${driverRows.length}'); - for (final r in driverRows) { - print(' Driver ID: ${r['driver_id']}, Account: ${r['account_id']}, Name: "${r['full_name']}"'); - } - - print('Co-Driver Profiles for Max Freeman: ${codriverRows.length}'); - for (final r in codriverRows) { - print(' Co-Driver ID: ${r['codriver_id']}, Account: ${r['account_id']}, Name: "${r['full_name']}"'); - } - - final driverId = driverRows.isNotEmpty ? driverRows.first['driver_id'] : null; - final codriverId = codriverRows.isNotEmpty ? codriverRows.first['codriver_id'] : null; - - // Raw Truth Query for Driver participation - final rawDriverParticipation = driverId != null - ? await db.query(''' - SELECT DISTINCT - re.event_id, - re.event_name, - re.country, - YEAR(re.start_date) AS year, - 'driver' AS role - FROM rally_entry_list el - JOIN rally_sub_events rse ON el.sub_event_id = rse.sub_event_id - JOIN rally_events re ON rse.event_id = re.event_id - WHERE el.user_driver_id = '$driverId' - ORDER BY re.start_date DESC - ''') - : >[]; - - // Raw Truth Query for Co-Driver participation - final rawCodriverParticipation = codriverId != null - ? await db.query(''' - SELECT DISTINCT - re.event_id, - re.event_name, - re.country, - YEAR(re.start_date) AS year, - 'codriver' AS role - FROM rally_entry_list el - JOIN rally_sub_events rse ON el.sub_event_id = rse.sub_event_id - JOIN rally_events re ON rse.event_id = re.event_id - WHERE el.user_co_driver_id = '$codriverId' - ORDER BY re.start_date DESC - ''') - : >[]; - - // Combined Raw Truth (Distinct Events) - final allRawEvents = >{}; - for (final r in rawDriverParticipation) { - allRawEvents[r['event_id'].toString()] = r; - } - for (final r in rawCodriverParticipation) { - allRawEvents[r['event_id'].toString()] = r; - } - - print('\nRAW DB TRUTH - Driver Events (${rawDriverParticipation.length}):'); - for (final r in rawDriverParticipation) { - print(' [Driver] ${r['event_id']} | ${r['event_name']} (${r['year']}, ${r['country']})'); - } - - print('\nRAW DB TRUTH - Co-Driver Events (${rawCodriverParticipation.length}):'); - for (final r in rawCodriverParticipation) { - print(' [Co-Driver] ${r['event_id']} | ${r['event_name']} (${r['year']}, ${r['country']})'); - } - - print('\nRAW DB TRUTH - Total Distinct Participation Events: ${allRawEvents.length}'); - - // Compare against SearchRepository.searchDriverRallies for PersonRole.any, driver, coDriver - print('\n--- TESTING REPOSITORY EXECUTION ---'); - - // 1. PersonRole.any - final repoAnyRes = await repo.searchDriverRallies( - SearchQuery( - intent: SearchIntent.searchDriverRallies, - driverName: 'Max Freeman', - personRole: PersonRole.any, - ), - ); - print('\nRepository searchDriverRallies(role: PersonRole.any):'); - print(' Found: ${repoAnyRes.results.length} rallies (Total Count: ${repoAnyRes.totalCount})'); - final repoAnyIds = repoAnyRes.results.map((r) => r.rallyId).toSet(); - for (final r in repoAnyRes.results) { - print(' - ${r.rallyId} | ${r.eventName} (${r.year}, ${r.country}) [Role: ${r.role}]'); - } - - // 2. PersonRole.driver - final repoDriverRes = await repo.searchDriverRallies( - SearchQuery( - intent: SearchIntent.searchDriverRallies, - driverName: 'Max Freeman', - personRole: PersonRole.driver, - ), - ); - print('\nRepository searchDriverRallies(role: PersonRole.driver):'); - print(' Found: ${repoDriverRes.results.length} rallies (Total Count: ${repoDriverRes.totalCount})'); - final repoDriverIds = repoDriverRes.results.map((r) => r.rallyId).toSet(); - for (final r in repoDriverRes.results) { - print(' - ${r.rallyId} | ${r.eventName} (${r.year}, ${r.country}) [Role: ${r.role}]'); - } - - // 3. PersonRole.coDriver - final repoCodriverRes = await repo.searchDriverRallies( - SearchQuery( - intent: SearchIntent.searchDriverRallies, - driverName: 'Max Freeman', - personRole: PersonRole.coDriver, - ), - ); - print('\nRepository searchDriverRallies(role: PersonRole.coDriver):'); - print(' Found: ${repoCodriverRes.results.length} rallies (Total Count: ${repoCodriverRes.totalCount})'); - final repoCodriverIds = repoCodriverRes.results.map((r) => r.rallyId).toSet(); - for (final r in repoCodriverRes.results) { - print(' - ${r.rallyId} | ${r.eventName} (${r.year}, ${r.country}) [Role: ${r.role}]'); - } - - // Compute exact diffs - final expectedDriverIds = rawDriverParticipation.map((r) => r['event_id'].toString()).toSet(); - final expectedCodriverIds = rawCodriverParticipation.map((r) => r['event_id'].toString()).toSet(); - final expectedAnyIds = allRawEvents.keys.toSet(); - - print('\n--- EXACT DIFF REPORT FOR MAX FREEMAN ---'); - print('ANY Role:'); - print(' Expected (${expectedAnyIds.length}): $expectedAnyIds'); - print(' Actual (${repoAnyIds.length}): $repoAnyIds'); - print(' Missing in Repo: ${expectedAnyIds.difference(repoAnyIds)}'); - print(' Extra in Repo: ${repoAnyIds.difference(expectedAnyIds)}'); - - print('DRIVER Role:'); - print(' Expected (${expectedDriverIds.length}): $expectedDriverIds'); - print(' Actual (${repoDriverIds.length}): $repoDriverIds'); - print(' Missing in Repo: ${expectedDriverIds.difference(repoDriverIds)}'); - print(' Extra in Repo: ${repoDriverIds.difference(expectedDriverIds)}'); - - print('CO-DRIVER Role:'); - print(' Expected (${expectedCodriverIds.length}): $expectedCodriverIds'); - print(' Actual (${repoCodriverIds.length}): $repoCodriverIds'); - print(' Missing in Repo: ${expectedCodriverIds.difference(repoCodriverIds)}'); - print(' Extra in Repo: ${repoCodriverIds.difference(expectedCodriverIds)}'); - }); - - test('3. Driver-Only / Co-Driver-Only / Dual-Role Representative Entities', () async { - print('\n================================================================'); - print('SECTION 3: REPRESENTATIVE ROLE ENTITIES AUDIT'); - print('================================================================'); - - // Find person who is Driver ONLY - final driverOnlyQuery = await db.query(''' - SELECT d.driver_id, d.full_name, COUNT(DISTINCT el.sub_event_id) as event_count - FROM user_driver_profile d - JOIN rally_entry_list el ON d.driver_id = el.user_driver_id - LEFT JOIN user_codriver_profile cd ON d.account_id = cd.account_id - WHERE cd.codriver_id IS NULL OR cd.codriver_id NOT IN (SELECT user_co_driver_id FROM rally_entry_list WHERE user_co_driver_id IS NOT NULL) - GROUP BY d.driver_id, d.full_name - HAVING event_count >= 3 - LIMIT 3; - '''); - print('\nRepresentative Driver-Only Persons:'); - for (final r in driverOnlyQuery) { - print(' ${r['full_name']} (Driver ID: ${r['driver_id']}, Events: ${r['event_count']})'); - } - - // Find person who is Co-Driver ONLY - final codriverOnlyQuery = await db.query(''' - SELECT cd.codriver_id, cd.full_name, COUNT(DISTINCT el.sub_event_id) as event_count - FROM user_codriver_profile cd - JOIN rally_entry_list el ON cd.codriver_id = el.user_co_driver_id - LEFT JOIN user_driver_profile d ON cd.account_id = d.account_id - WHERE d.driver_id IS NULL OR d.driver_id NOT IN (SELECT user_driver_id FROM rally_entry_list WHERE user_driver_id IS NOT NULL) - GROUP BY cd.codriver_id, cd.full_name - HAVING event_count >= 3 - LIMIT 3; - '''); - print('\nRepresentative Co-Driver-Only Persons:'); - for (final r in codriverOnlyQuery) { - print(' ${r['full_name']} (Co-Driver ID: ${r['codriver_id']}, Events: ${r['event_count']})'); - } - - // Find person who is in BOTH roles with entries in both - final dualRoleQuery = await db.query(''' - SELECT - d.full_name, - d.driver_id, cd.codriver_id, - (SELECT COUNT(DISTINCT sub_event_id) FROM rally_entry_list WHERE user_driver_id = d.driver_id) AS driver_events, - (SELECT COUNT(DISTINCT sub_event_id) FROM rally_entry_list WHERE user_co_driver_id = cd.codriver_id) AS codriver_events - FROM user_driver_profile d - JOIN user_codriver_profile cd ON d.account_id = cd.account_id - WHERE d.driver_id IN (SELECT user_driver_id FROM rally_entry_list) - AND cd.codriver_id IN (SELECT user_co_driver_id FROM rally_entry_list) - LIMIT 5; - '''); - print('\nRepresentative Dual-Role Persons:'); - for (final r in dualRoleQuery) { - print(' ${r['full_name']} (Driver Events: ${r['driver_events']}, Co-Driver Events: ${r['codriver_events']})'); - } - - // Test Driver-Only Case - if (driverOnlyQuery.isNotEmpty) { - final dName = driverOnlyQuery.first['full_name']?.toString() ?? ''; - final dId = driverOnlyQuery.first['driver_id']; - final rawTruth = await db.query(''' - SELECT DISTINCT re.event_id, re.event_name FROM rally_entry_list el - JOIN rally_sub_events rse ON el.sub_event_id = rse.sub_event_id - JOIN rally_events re ON rse.event_id = re.event_id - WHERE el.user_driver_id = '$dId' - '''); - final rawIds = rawTruth.map((r) => r['event_id'].toString()).toSet(); - final repoRes = await repo.searchDriverRallies( - SearchQuery( - intent: SearchIntent.searchDriverRallies, - driverName: dName, - personRole: PersonRole.any, - ), - ); - final repoIds = repoRes.results.map((r) => r.rallyId).toSet(); - - print('\nDriver-Only Audit for "$dName":'); - print(' Raw Truth Events: ${rawIds.length}, Repo Events: ${repoIds.length}'); - print(' Missing: ${rawIds.difference(repoIds)}, Extra: ${repoIds.difference(rawIds)}'); - } - - // Test Co-Driver-Only Case - if (codriverOnlyQuery.isNotEmpty) { - final cdName = codriverOnlyQuery.first['full_name']?.toString() ?? ''; - final cdId = codriverOnlyQuery.first['codriver_id']; - final rawTruth = await db.query(''' - SELECT DISTINCT re.event_id, re.event_name FROM rally_entry_list el - JOIN rally_sub_events rse ON el.sub_event_id = rse.sub_event_id - JOIN rally_events re ON rse.event_id = re.event_id - WHERE el.user_co_driver_id = '$cdId' - '''); - final rawIds = rawTruth.map((r) => r['event_id'].toString()).toSet(); - final repoRes = await repo.searchDriverRallies( - SearchQuery( - intent: SearchIntent.searchDriverRallies, - driverName: cdName, - personRole: PersonRole.any, - ), - ); - final repoIds = repoRes.results.map((r) => r.rallyId).toSet(); - - print('\nCo-Driver-Only Audit for "$cdName":'); - print(' Raw Truth Events: ${rawIds.length}, Repo Events: ${repoIds.length}'); - print(' Missing: ${rawIds.difference(repoIds)}, Extra: ${repoIds.difference(rawIds)}'); - } - }); - - test('4. Uploader / Fan Username Mapping Audit', () async { - print('\n================================================================'); - print('SECTION 4: UPLOADER / FAN USERNAME MAPPING AUDIT'); - print('================================================================'); - - // Query raw relation between rally_videos, user_fan_profile, user_account - final uploaderQuery = await db.query(''' - SELECT - v.uploader_user_id, - f.fan_id, - f.account_id, - f.full_name AS fan_full_name, - f.profile_picture, - a.id AS account_id_from_acc, - a.user_name, - a.email, - COUNT(v.id) AS video_count - FROM rally_videos v - LEFT JOIN user_fan_profile f ON v.uploader_user_id = f.fan_id - LEFT JOIN user_account a ON f.account_id = a.id - GROUP BY v.uploader_user_id, f.fan_id, f.account_id, f.full_name, f.profile_picture, a.id, a.user_name, a.email - ORDER BY video_count DESC - LIMIT 10; - '''); - - print('\nTop 10 Video Uploaders in DB:'); - for (final r in uploaderQuery) { - print(' Uploader ID: ${r['uploader_user_id']} | Videos: ${r['video_count']} | user_name: "${r['user_name']}" | fan_full_name: "${r['fan_full_name']}" | email: "${r['email']}"'); - } - - // Check SearchRepository.getTopUploaders - final repoTopUploaders = await repo.getTopUploaders( - SearchQuery(intent: SearchIntent.getTopUploaders), - ); - print('\nRepository getTopUploaders (${repoTopUploaders.results.length} uploaders):'); - for (final u in repoTopUploaders.results.take(5)) { - print(' Name: "${u.uploaderName}", Videos: ${u.uploadCount}, ID: ${u.uploaderId}, Avatar: ${u.profilePicture}'); - } - }); - }); -} diff --git a/test/eval/smoke_test.dart b/test/eval/smoke_test.dart index d0750c2..c8f4be5 100644 --- a/test/eval/smoke_test.dart +++ b/test/eval/smoke_test.dart @@ -16,10 +16,21 @@ void main() { print('🏎️ AI RALLY SEARCH — OPENAI LIVE SMOKE TEST (5 BENCHMARK CASES)'); print('================================================================'); - // Load .env + // Secrets are no longer bundled as an app asset. Load a developer's local + // on-disk .env (if present) for this live test; otherwise skip. final envFile = File('.env'); if (envFile.existsSync()) { - await dotenv.load(fileName: '.env'); + dotenv.loadFromString( + envString: envFile.readAsStringSync(), + isOptional: true, + ); + } + if ((dotenv.maybeGet('OPENAI_API_KEY') ?? '').isEmpty) { + markTestSkipped( + 'Live OpenAI test skipped: OPENAI_API_KEY unavailable. Provide a local ' + '.env to run this test (keys are no longer shipped in the app bundle).', + ); + return; } final parser = OpenAIQueryParser(); diff --git a/test/eval/structured_parity_runner_test.dart b/test/eval/structured_parity_runner_test.dart deleted file mode 100644 index dbf3023..0000000 --- a/test/eval/structured_parity_runner_test.dart +++ /dev/null @@ -1,12 +0,0 @@ -import 'package:flutter_test/flutter_test.dart'; -import '../../bin/run_structured_parity.dart' as runner; - -void main() { - TestWidgetsFlutterBinding.ensureInitialized(); - test('execute shared deterministic parity fixtures', () async { - const fixtures=String.fromEnvironment('PARITY_FIXTURES'); - const output=String.fromEnvironment('PARITY_OUTPUT'); - expect(fixtures,isNotEmpty); expect(output,isNotEmpty); - await runner.main([fixtures,output]); - }, timeout: const Timeout(Duration(minutes: 5))); -} diff --git a/test/eval/video_person_semantics_audit_test.dart b/test/eval/video_person_semantics_audit_test.dart deleted file mode 100644 index ba2177c..0000000 --- a/test/eval/video_person_semantics_audit_test.dart +++ /dev/null @@ -1,351 +0,0 @@ -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:flutter_test/flutter_test.dart'; -import 'package:ai_rally_search/models/search_intent.dart'; -import 'package:ai_rally_search/models/search_query.dart'; -import 'package:ai_rally_search/models/search_results.dart'; -import 'package:ai_rally_search/models/result_referent_context.dart'; -import 'package:ai_rally_search/services/database_service.dart'; -import 'package:ai_rally_search/services/search_repository.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/database_entity_resolver.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/entity_lookup_repository.dart'; - -void main() { - late DatabaseService dbService; - late SearchRepository searchRepo; - late DatabaseEntityLookupRepository lookupRepo; - late DatabaseEntityResolver resolver; - - setUpAll(() async { - await dotenv.load(fileName: '.env'); - dbService = DatabaseService(); - searchRepo = SearchRepository(dbService: dbService); - lookupRepo = DatabaseEntityLookupRepository(dbService: dbService); - resolver = DatabaseEntityResolver(repository: lookupRepo); - }); - - group('1. Max Freeman (Co-Driver Only) Video & Action Semantics Audit', () { - test('Max Freeman: searchDriverVideos & searchVideoActions raw truth vs repository', () async { - // 1. Raw DB Truth for Max Freeman Videos (via rally_video_metadata -> rally_entry_list -> user_codriver_profile) - const rawMaxFreemanVideosSql = ''' - SELECT DISTINCT rv.id AS video_id - FROM rally_videos rv - INNER JOIN rally_video_metadata vm ON rv.id = vm.video_id - INNER JOIN rally_entry_list el ON vm.entry_list_id = el.id - INNER JOIN user_codriver_profile cdp ON el.user_co_driver_id = cdp.codriver_id - INNER JOIN rally_streams rs ON rv.id = rs.video_id - WHERE LOWER(cdp.full_name) LIKE '%max freeman%' - AND rs.on_demand_url IS NOT NULL AND rs.on_demand_url != '' - AND (rs.video_type IS NULL OR rs.video_type != 'instantReplay') - '''; - final rawVideoRows = await dbService.query(rawMaxFreemanVideosSql); - final expectedVideoIds = rawVideoRows.map((r) => int.parse(r['video_id'].toString())).toSet(); - - // 2. Raw DB Truth for Max Freeman Video Actions (via rally_video_metadata -> rally_entry_list -> user_codriver_profile) - const rawMaxFreemanActionsSql = ''' - SELECT DISTINCT vm.id AS action_id - FROM rally_video_metadata vm - INNER JOIN rally_video_actions va ON vm.action_id = va.id - INNER JOIN rally_streams rs ON vm.video_id = rs.video_id - INNER JOIN rally_entry_list el ON vm.entry_list_id = el.id - INNER JOIN user_codriver_profile cdp ON el.user_co_driver_id = cdp.codriver_id - WHERE LOWER(cdp.full_name) LIKE '%max freeman%' - AND rs.on_demand_url IS NOT NULL AND rs.on_demand_url != '' - AND (rs.video_type IS NULL OR rs.video_type != 'instantReplay') - '''; - final rawActionRows = await dbService.query(rawMaxFreemanActionsSql); - final expectedActionIds = rawActionRows.map((r) => int.parse(r['action_id'].toString())).toSet(); - - // Check DRIVER role (Must return 0 for both videos and actions since Max Freeman is strictly a co-driver) - final qDriverVid = SearchQuery( - intent: SearchIntent.searchDriverVideos, - driverNames: ['Max Freeman'], - personRole: PersonRole.driver, - ); - final resDriverVid = await resolver.resolve(qDriverVid); - final respDriverVid = await searchRepo.search(resDriverVid.resolvedQuery!); - expect(respDriverVid.totalCount, 0); - expect(respDriverVid.results.isEmpty, isTrue); - - final qDriverAct = SearchQuery( - intent: SearchIntent.searchVideoActions, - driverNames: ['Max Freeman'], - personRole: PersonRole.driver, - ); - final resDriverAct = await resolver.resolve(qDriverAct); - final respDriverAct = await searchRepo.search(resDriverAct.resolvedQuery!); - expect(respDriverAct.totalCount, 0); - expect(respDriverAct.results.isEmpty, isTrue); - - // Check CO_DRIVER role - final qCodriverVid = SearchQuery( - intent: SearchIntent.searchDriverVideos, - driverNames: ['Max Freeman'], - personRole: PersonRole.coDriver, - limit: 100, - ); - final resCodriverVid = await resolver.resolve(qCodriverVid); - final respCodriverVid = await searchRepo.search(resCodriverVid.resolvedQuery!); - expect(respCodriverVid.totalCount, expectedVideoIds.length); - final actualVidIds = respCodriverVid.results.map((r) => r.videoId).toSet(); - expect(actualVidIds, equals(expectedVideoIds)); - - final qCodriverAct = SearchQuery( - intent: SearchIntent.searchVideoActions, - driverNames: ['Max Freeman'], - personRole: PersonRole.coDriver, - limit: 100, - ); - final resCodriverAct = await resolver.resolve(qCodriverAct); - final respCodriverAct = await searchRepo.search(resCodriverAct.resolvedQuery!); - expect(respCodriverAct.totalCount, expectedActionIds.length); - final actualActIds = respCodriverAct.results.map((r) => r.id).toSet(); - expect(actualActIds, equals(expectedActionIds)); - - // Check ANY role - final qAnyVid = SearchQuery( - intent: SearchIntent.searchDriverVideos, - driverNames: ['Max Freeman'], - personRole: PersonRole.any, - limit: 100, - ); - final resAnyVid = await resolver.resolve(qAnyVid); - final respAnyVid = await searchRepo.search(resAnyVid.resolvedQuery!); - expect(respAnyVid.totalCount, expectedVideoIds.length); - final actualAnyVidIds = respAnyVid.results.map((r) => r.videoId).toSet(); - expect(actualAnyVidIds, equals(expectedVideoIds)); - - final qAnyAct = SearchQuery( - intent: SearchIntent.searchVideoActions, - driverNames: ['Max Freeman'], - personRole: PersonRole.any, - limit: 100, - ); - final resAnyAct = await resolver.resolve(qAnyAct); - final respAnyAct = await searchRepo.search(resAnyAct.resolvedQuery!); - expect(respAnyAct.totalCount, expectedActionIds.length); - final actualAnyActIds = respAnyAct.results.map((r) => r.id).toSet(); - expect(actualAnyActIds, equals(expectedActionIds)); - }); - }); - - group('2. Driver-Only Golden Case (Josh Moffett)', () { - test('Josh Moffett: Videos & Actions comparison for DRIVER vs CO_DRIVER vs ANY', () async { - // 1. Raw DB truth for Josh Moffett as DRIVER - const rawJoshDriverVideosSql = ''' - SELECT DISTINCT rv.id AS video_id - FROM rally_videos rv - INNER JOIN rally_video_metadata vm ON rv.id = vm.video_id - INNER JOIN rally_entry_list el ON vm.entry_list_id = el.id - INNER JOIN user_driver_profile dp ON el.user_driver_id = dp.driver_id - INNER JOIN rally_streams rs ON rv.id = rs.video_id - WHERE LOWER(dp.full_name) LIKE '%josh moffett%' - AND rs.on_demand_url IS NOT NULL AND rs.on_demand_url != '' - AND (rs.video_type IS NULL OR rs.video_type != 'instantReplay') - '''; - final rawDriverVidRows = await dbService.query(rawJoshDriverVideosSql); - final expectedDriverVidIds = rawDriverVidRows.map((r) => int.parse(r['video_id'].toString())).toSet(); - - const rawJoshDriverActionsSql = ''' - SELECT DISTINCT vm.id AS action_id - FROM rally_video_metadata vm - INNER JOIN rally_video_actions va ON vm.action_id = va.id - INNER JOIN rally_streams rs ON vm.video_id = rs.video_id - INNER JOIN rally_entry_list el ON vm.entry_list_id = el.id - INNER JOIN user_driver_profile dp ON el.user_driver_id = dp.driver_id - WHERE LOWER(dp.full_name) LIKE '%josh moffett%' - AND rs.on_demand_url IS NOT NULL AND rs.on_demand_url != '' - AND (rs.video_type IS NULL OR rs.video_type != 'instantReplay') - '''; - final rawDriverActRows = await dbService.query(rawJoshDriverActionsSql); - final expectedDriverActIds = rawDriverActRows.map((r) => int.parse(r['action_id'].toString())).toSet(); - - // 2. Raw DB truth for Josh Moffett as CO_DRIVER - const rawJoshCodriverVideosSql = ''' - SELECT DISTINCT rv.id AS video_id - FROM rally_videos rv - INNER JOIN rally_video_metadata vm ON rv.id = vm.video_id - INNER JOIN rally_entry_list el ON vm.entry_list_id = el.id - INNER JOIN user_codriver_profile cdp ON el.user_co_driver_id = cdp.codriver_id - INNER JOIN rally_streams rs ON rv.id = rs.video_id - WHERE LOWER(cdp.full_name) LIKE '%josh moffett%' - AND rs.on_demand_url IS NOT NULL AND rs.on_demand_url != '' - AND (rs.video_type IS NULL OR rs.video_type != 'instantReplay') - '''; - final rawCodriverVidRows = await dbService.query(rawJoshCodriverVideosSql); - final expectedCodriverVidIds = rawCodriverVidRows.map((r) => int.parse(r['video_id'].toString())).toSet(); - - // DRIVER query - final qDriver = SearchQuery( - intent: SearchIntent.searchDriverVideos, - driverNames: ['Josh Moffett'], - personRole: PersonRole.driver, - limit: 500, - ); - final resDriver = await resolver.resolve(qDriver); - final respDriver = await searchRepo.search(resDriver.resolvedQuery!); - expect(respDriver.totalCount, expectedDriverVidIds.length); - final actualDriverVidIds = respDriver.results.map((r) => r.videoId).toSet(); - expect(actualDriverVidIds, equals(expectedDriverVidIds)); - - // CO_DRIVER query - final qCodriver = SearchQuery( - intent: SearchIntent.searchDriverVideos, - driverNames: ['Josh Moffett'], - personRole: PersonRole.coDriver, - limit: 500, - ); - final resCodriver = await resolver.resolve(qCodriver); - final respCodriver = await searchRepo.search(resCodriver.resolvedQuery!); - expect(respCodriver.totalCount, expectedCodriverVidIds.length); - - // ANY query - final qAny = SearchQuery( - intent: SearchIntent.searchDriverVideos, - driverNames: ['Josh Moffett'], - personRole: PersonRole.any, - limit: 500, - ); - final resAny = await resolver.resolve(qAny); - final respAny = await searchRepo.search(resAny.resolvedQuery!); - final expectedAllVidIds = {...expectedDriverVidIds, ...expectedCodriverVidIds}; - expect(respAny.totalCount, expectedAllVidIds.length); - final actualAnyVidIds = respAny.results.map((r) => r.videoId).toSet(); - expect(actualAnyVidIds, equals(expectedAllVidIds)); - }); - }); - - group('3. Dual-Role Case (Account 419633b1-56ca-483a-8c10-0141d7cc3092)', () { - test('Chris Melly / Melly Chris: Exact video and action IDs per role', () async { - const accId = '419633b1-56ca-483a-8c10-0141d7cc3092'; - - // Raw truth queries - final rawDriverVidSql = ''' - SELECT DISTINCT rv.id AS video_id - FROM rally_videos rv - INNER JOIN rally_video_metadata vm ON rv.id = vm.video_id - INNER JOIN rally_entry_list el ON vm.entry_list_id = el.id - INNER JOIN user_driver_profile dp ON el.user_driver_id = dp.driver_id - INNER JOIN rally_streams rs ON rv.id = rs.video_id - WHERE dp.account_id = '$accId' - AND rs.on_demand_url IS NOT NULL AND rs.on_demand_url != '' - AND (rs.video_type IS NULL OR rs.video_type != 'instantReplay') - '''; - final rawDriverVidRows = await dbService.query(rawDriverVidSql); - final expectedDriverVidIds = rawDriverVidRows.map((r) => int.parse(r['video_id'].toString())).toSet(); - - final rawCodriverVidSql = ''' - SELECT DISTINCT rv.id AS video_id - FROM rally_videos rv - INNER JOIN rally_video_metadata vm ON rv.id = vm.video_id - INNER JOIN rally_entry_list el ON vm.entry_list_id = el.id - INNER JOIN user_codriver_profile cdp ON el.user_co_driver_id = cdp.codriver_id - INNER JOIN rally_streams rs ON rv.id = rs.video_id - WHERE cdp.account_id = '$accId' - AND rs.on_demand_url IS NOT NULL AND rs.on_demand_url != '' - AND (rs.video_type IS NULL OR rs.video_type != 'instantReplay') - '''; - final rawCodriverVidRows = await dbService.query(rawCodriverVidSql); - final expectedCodriverVidIds = rawCodriverVidRows.map((r) => int.parse(r['video_id'].toString())).toSet(); - - final expectedAllVidIds = {...expectedDriverVidIds, ...expectedCodriverVidIds}; - - // 1. DRIVER role - final qDriver = SearchQuery( - intent: SearchIntent.searchDriverVideos, - driverNames: ['melly chris'], - personRole: PersonRole.driver, - limit: 50, - ); - final resDriver = await resolver.resolve(qDriver); - final respDriver = await searchRepo.search(resDriver.resolvedQuery!); - expect(respDriver.totalCount, expectedDriverVidIds.length); - final actualDriverVidIds = respDriver.results.map((r) => r.videoId).toSet(); - expect(actualDriverVidIds, equals(expectedDriverVidIds)); - - // 2. CO_DRIVER role - final qCodriver = SearchQuery( - intent: SearchIntent.searchDriverVideos, - driverNames: ['melly chris'], - personRole: PersonRole.coDriver, - limit: 200, - ); - final resCodriver = await resolver.resolve(qCodriver); - final respCodriver = await searchRepo.search(resCodriver.resolvedQuery!); - expect(respCodriver.totalCount, expectedCodriverVidIds.length); - final actualCodriverVidIds = respCodriver.results.map((r) => r.videoId).toSet(); - expect(actualCodriverVidIds, equals(expectedCodriverVidIds)); - - // 3. ANY role - final qAny = SearchQuery( - intent: SearchIntent.searchDriverVideos, - driverNames: ['melly chris'], - personRole: PersonRole.any, - limit: 200, - ); - final resAny = await resolver.resolve(qAny); - final respAny = await searchRepo.search(resAny.resolvedQuery!); - expect(respAny.totalCount, expectedAllVidIds.length); - final actualAnyVidIds = respAny.results.map((r) => r.videoId).toSet(); - expect(actualAnyVidIds, equals(expectedAllVidIds)); - }); - }); - - group('4. Deduplication & Pagination Integrity', () { - test('Video actions pagination has zero duplicate rows and correct totalCount', () async { - final qPage1 = SearchQuery( - intent: SearchIntent.searchVideoActions, - actionTypes: ['jump'], - limit: 10, - offset: 0, - ); - final respPage1 = await searchRepo.search(qPage1); - final page1Ids = respPage1.results.map((r) => r.id).toList(); - expect(page1Ids.toSet().length, page1Ids.length, reason: 'Page 1 must have no duplicate IDs'); - - if (respPage1.totalCount > 10) { - final qPage2 = SearchQuery( - intent: SearchIntent.searchVideoActions, - actionTypes: ['jump'], - limit: 10, - offset: 10, - ); - final respPage2 = await searchRepo.search(qPage2); - final page2Ids = respPage2.results.map((r) => r.id).toList(); - expect(page2Ids.toSet().length, page2Ids.length, reason: 'Page 2 must have no duplicate IDs'); - - final overlap = page1Ids.toSet().intersection(page2Ids.toSet()); - expect(overlap, isEmpty, reason: 'Page 1 and Page 2 must not have overlapping IDs'); - } - }); - }); - - group('5. Conversational Follow-Up Semantics Distinction', () { - test('Distinguishes "videos from those rallies" (event referents) vs "videos of X" (person filter)', () async { - // Step 1: "Which rallies did Max Freeman co-drive in?" - final qRallies = SearchQuery( - intent: SearchIntent.searchDriverRallies, - driverNames: ['Max Freeman'], - personRole: PersonRole.coDriver, - ); - final resRallies = await resolver.resolve(qRallies); - final respRallies = await searchRepo.search(resRallies.resolvedQuery!); - - final contextAfterRallies = ResultReferentContext.fromSearchResponse( - respRallies, - queryDriver: 'Max Freeman', - queryPersonRole: PersonRole.coDriver, - queryRallies: respRallies.results.map((r) => r.eventName.toString()).toList(), - ); - - // Verification A: Follow-up "Show videos from those rallies" - // Expected: Uses inherited event referents (activeRallies), NOT direct person filter on video crew - expect(contextAfterRallies.activeRallies, isNotEmpty); - expect(contextAfterRallies.activeRallies.length, greaterThan(1)); - - // Verification B: "Show videos of Max Freeman" - // Expected: Uses person filter (driverNames: ["Max Freeman"], personRole: CO_DRIVER) - expect(contextAfterRallies.activeDriver, 'Max Freeman'); - expect(contextAfterRallies.activePersonRole, PersonRole.coDriver); - }); - }); -} diff --git a/test/gemini_live_test.dart b/test/gemini_live_test.dart index fc182b2..52c64b7 100644 --- a/test/gemini_live_test.dart +++ b/test/gemini_live_test.dart @@ -13,7 +13,22 @@ void main() { TestWidgetsFlutterBinding.ensureInitialized(); HttpOverrides.global = null; - await dotenv.load(fileName: '.env'); + // Secrets are no longer bundled as an app asset. Load a developer's local + // on-disk .env (if present) for this live test; otherwise skip. + final envFile = File('.env'); + if (envFile.existsSync()) { + dotenv.loadFromString( + envString: envFile.readAsStringSync(), + isOptional: true, + ); + } + if ((dotenv.maybeGet('GEMINI_API_KEY') ?? '').isEmpty) { + markTestSkipped( + 'Live Gemini test skipped: GEMINI_API_KEY unavailable. Provide a local ' + '.env to run this test (keys are no longer shipped in the app bundle).', + ); + return; + } print('Loaded .env successfully'); final config = LlmConfig.fromEnvironment(defaultProvider: LlmProvider.gemini); diff --git a/test/latency/latency_test_fixtures.dart b/test/latency/latency_test_fixtures.dart new file mode 100644 index 0000000..0758c66 --- /dev/null +++ b/test/latency/latency_test_fixtures.dart @@ -0,0 +1,36 @@ +import 'package:ai_rally_search/services/offline/offline_database.dart'; +import 'package:ai_rally_search/services/offline/offline_search_engine.dart'; + +/// A minimal snapshot with one Irish rally, distinct by name from anything the +/// fake online repository returns. +Map irelandSnapshot() => { + 'schema_version': 1, + 'data_version': 'v1', + 'snapshot_id': '1-v1-core', + 'segment': 'core', + 'generated_at': '2026-08-30T00:00:00Z', + 'rallies': >[ + { + 'event_id': 'ev1', + 'event_name': 'Rally Alpha 2025', + 'country': 'Ireland', + 'city': 'Cork', + 'year': 2025, + 'start_date': '2025-05-01', + 'end_date': null, + 'status': null, + 'stages_count': 3, + }, + ], + 'people': const [], + 'stages': const [], + 'participation': const [], + 'final_results': const [], + 'driver_wins': const [], + 'uploader_stats': const [], + 'video_meta': const [], + 'video_actions': const [], + }; + +Future offlineEngine(OfflineDatabase db) => + OfflineSearchEngine.create(db); diff --git a/test/latency/progressive_fallback_screen_test.dart b/test/latency/progressive_fallback_screen_test.dart new file mode 100644 index 0000000..9073427 --- /dev/null +++ b/test/latency/progressive_fallback_screen_test.dart @@ -0,0 +1,476 @@ +import 'dart:async'; +import 'dart:io'; + +import 'package:ai_rally_search/l10n/generated/app_localizations.dart'; +import 'package:ai_rally_search/models/entity_candidate.dart'; +import 'package:ai_rally_search/models/search_intent.dart'; +import 'package:ai_rally_search/models/search_query.dart'; +import 'package:ai_rally_search/models/search_results.dart'; +import 'package:ai_rally_search/models/video_action.dart'; +import 'package:ai_rally_search/screens/general_search_screen.dart'; +import 'package:ai_rally_search/services/latency/latency_policy.dart'; +import 'package:ai_rally_search/services/latency/search_latency_coordinator.dart'; +import 'package:ai_rally_search/services/latency/search_telemetry.dart'; +import 'package:ai_rally_search/services/llm/entity_resolution/entity_resolver.dart'; +import 'package:ai_rally_search/services/llm/llm_provider_config.dart'; +import 'package:ai_rally_search/services/llm/llm_query_parser.dart'; +import 'package:ai_rally_search/services/llm/natural_language_search_service.dart'; +import 'package:ai_rally_search/services/llm/query_parse_result.dart'; +import 'package:ai_rally_search/services/offline/offline_database.dart'; +import 'package:ai_rally_search/services/search_repository.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +import 'latency_test_fixtures.dart'; + +/// A parser whose completion the test controls, so the online turn can be made +/// to land on either side of the fallback budget. +class _ControlledParser implements LlmQueryParser { + final Map> _pending = {}; + int calls = 0; + + @override + LlmProvider get provider => LlmProvider.mock; + + @override + Future parse(String userQuery, {SearchContext? context}) { + calls++; + return (_pending[userQuery] ??= Completer()).future; + } + + bool get isWaiting => _pending.values.any((c) => !c.isCompleted); + + /// Resolves the pending call for [text]. + /// + /// The service normalizes a query before parsing it ("rallies in ireland" + /// arrives as "rallies in Ireland"), so tests are matched case-insensitively + /// against what the parser actually received rather than against the raw + /// text the test typed. + Completer _pendingFor(String text) { + final key = _pending.keys.firstWhere( + (k) => k.toLowerCase() == text.toLowerCase(), + orElse: () => throw StateError( + 'the parser was never called for "$text"; saw ${_pending.keys.toList()}', + ), + ); + return _pending[key]!; + } + + void completeWith(String text, SearchQuery query) { + _pendingFor(text).complete(QueryParseResult(query: query)); + } + + void failWith(String text, Object error) { + _pendingFor(text).completeError(error); + } + + void clarify(String text, String question) { + _pendingFor(text).complete( + QueryParseResult(requiresClarification: true, clarificationQuestion: question), + ); + } +} + +class _PassThroughResolver implements EntityResolver { + @override + Future resolve( + SearchQuery query, { + SearchContext? context, + }) async => + EntityResolutionResult.success(parsedQuery: query, resolvedQuery: query); +} + +/// Returns an online-only result that is visibly distinct from anything in the +/// local snapshot, so a swap between the two is unmistakable on screen. +class _OnlineOnlyRepository implements ISearchRepository { + @override + Future> search(SearchQuery query) async { + return SearchResponse( + intent: SearchIntent.searchRallies, + results: const [ + RallySearchResult( + eventId: 'online-1', + eventName: 'HQ Authoritative Rally 2026', + country: 'Ireland', + city: 'Cork', + stagesCount: 9, + ), + ], + totalCount: 1, + hasMore: false, + limit: query.limit, + offset: query.offset, + ); + } + + Future> _t(SearchQuery q) async => + SearchResponse( + intent: SearchIntent.searchRallies, + results: (await search(q)).results.cast(), + totalCount: 1, + hasMore: false, + limit: q.limit, + offset: q.offset, + ); + + @override + Future> searchRallies(SearchQuery q) => _t(q); + @override + Future> searchDriverRallies(SearchQuery q) => _t(q); + @override + Future> searchDriverWins(SearchQuery q) => _t(q); + @override + Future> getRallyResults(SearchQuery q) => _t(q); + @override + Future> getRallyTopFinishers(SearchQuery q) => _t(q); + @override + Future> searchVideoActions(SearchQuery q) => _t(q); + @override + Future> searchDriverVideos(SearchQuery q) => _t(q); + @override + Future> getTopUploaders(SearchQuery q) => _t(q); + @override + Future> getTopDriversByWins(SearchQuery q) => _t(q); +} + +class _Probe implements ConnectivityProbe { + final bool online; + const _Probe(this.online); + @override + Future isOnline() async => online; +} + +const _budget = Duration(milliseconds: 200); +const _query = 'rallies in ireland'; +const _onlineQuery = SearchQuery( + intent: SearchIntent.searchRallies, + countries: ['Ireland'], +); + +void main() { + setUpAll(sqfliteFfiInit); + + late _ControlledParser parser; + late InMemorySearchTelemetrySink telemetry; + + setUp(() { + parser = _ControlledParser(); + telemetry = InMemorySearchTelemetrySink(); + }); + + Future app({ + bool online = true, + bool withSnapshot = true, + Duration budget = _budget, + Key? key, + }) async { + // The shared in-memory path keeps its contents within an isolate, so the + // empty-snapshot case needs a file of its own to genuinely have none. + final OfflineDatabase db; + if (withSnapshot) { + db = await OfflineDatabase.open( + factory: databaseFactoryFfiNoIsolate, + path: inMemoryDatabasePath, + ); + await db.importSnapshot(irelandSnapshot()); + } else { + // Synchronous file work: real async I/O never completes inside a widget + // test's fake-async zone. + final dir = Directory.systemTemp.createTempSync('rally_no_snapshot'); + addTearDown(() => dir.deleteSync(recursive: true)); + db = await OfflineDatabase.open( + factory: databaseFactoryFfiNoIsolate, + path: '${dir.path}/empty.db', + ); + } + final engine = await offlineEngine(db); + return MaterialApp( + localizationsDelegates: AppLocalizations.localizationsDelegates, + supportedLocales: AppLocalizations.supportedLocales, + home: GeneralSearchScreen( + key: key, + offlineEngine: engine, + connectivityProbe: _Probe(online), + telemetrySink: telemetry, + latencyPolicy: LatencyPolicy( + onlineResultBudget: budget, + overallOnlineTimeout: const Duration(seconds: 5), + ), + nlSearchService: NaturalLanguageSearchService( + parser: parser, + entityResolver: _PassThroughResolver(), + repository: _OnlineOnlyRepository(), + ), + ), + ); + } + + Future submit(WidgetTester tester, [String text = _query]) async { + await tester.enterText(find.byType(TextField).first, text); + await tester.testTextInput.receiveAction(TextInputAction.search); + await tester.pump(); + } + + /// Advances time in slices so timers, futures and rebuilds all settle. + Future settle(WidgetTester tester, + {Duration by = const Duration(milliseconds: 60), int times = 6}) async { + for (var i = 0; i < times; i++) { + await tester.pump(by); + } + } + + testWidgets('1. online result inside the budget is shown as the online result', + (tester) async { + await tester.pumpWidget(await app()); + await settle(tester, times: 2); + await submit(tester); + + parser.completeWith(_query, _onlineQuery); + await settle(tester, by: const Duration(milliseconds: 20), times: 4); + + expect(find.text('HQ Authoritative Rally 2026'), findsOneWidget); + expect(find.textContaining('Taking the service road'), findsNothing); + expect(telemetry.records.single.resultSource, SearchResultSource.online); + expect(telemetry.records.single.fallbackTriggered, isFalse); + }); + + testWidgets('2. online past the budget with a valid local result falls back', + (tester) async { + await tester.pumpWidget(await app()); + await settle(tester, times: 2); + await submit(tester); + await settle(tester); + + expect(find.textContaining('Taking the service road'), findsOneWidget); + expect(find.text('Rally Alpha 2025'), findsOneWidget); + expect(find.text('HQ Authoritative Rally 2026'), findsNothing); + + final record = telemetry.records.single; + expect(record.resultSource, SearchResultSource.offlineFallback); + expect(record.fallbackTriggered, isTrue); + expect(record.fallbackTriggerMs, greaterThanOrEqualTo(_budget.inMilliseconds)); + expect(record.localParserCouldAnswer, isTrue); + expect(record.requestId, isNotEmpty); + + // The online request is still running: it was never cancelled. + expect(parser.isWaiting, isTrue); + }); + + testWidgets('3. online just before the budget produces no fallback', + (tester) async { + await tester.pumpWidget(await app(budget: const Duration(seconds: 2))); + await settle(tester, times: 2); + await submit(tester); + await tester.pump(const Duration(milliseconds: 900)); + + parser.completeWith(_query, _onlineQuery); + await settle(tester, by: const Duration(milliseconds: 40), times: 4); + + expect(find.text('HQ Authoritative Rally 2026'), findsOneWidget); + expect(find.textContaining('Taking the service road'), findsNothing); + expect(find.textContaining('HQ has fresh results'), findsNothing); + }); + + testWidgets( + '4. online just after the budget keeps local and offers the fresh result', + (tester) async { + await tester.pumpWidget(await app()); + await settle(tester, times: 2); + await submit(tester); + await settle(tester); + + expect(find.text('Rally Alpha 2025'), findsOneWidget); + + parser.completeWith(_query, _onlineQuery); + await settle(tester, by: const Duration(milliseconds: 40), times: 5); + + // Offered, not applied: the local result is still the one on screen. + expect(find.textContaining('HQ has fresh results'), findsOneWidget); + expect(find.text('Show latest'), findsOneWidget); + expect(find.text('Rally Alpha 2025'), findsOneWidget); + expect(find.text('HQ Authoritative Rally 2026'), findsNothing); + }); + + testWidgets('5. tapping "show latest" replaces local with the online result', + (tester) async { + await tester.pumpWidget(await app()); + await settle(tester, times: 2); + await submit(tester); + await settle(tester); + parser.completeWith(_query, _onlineQuery); + await settle(tester, by: const Duration(milliseconds: 40), times: 5); + + await tester.tap(find.text('Show latest')); + await settle(tester, by: const Duration(milliseconds: 40), times: 5); + + expect(find.text('HQ Authoritative Rally 2026'), findsOneWidget); + expect(find.text('Rally Alpha 2025'), findsNothing); + expect(find.textContaining('HQ has fresh results'), findsNothing); + expect(find.textContaining('Taking the service road'), findsNothing); + }); + + testWidgets('5b. repeated "show latest" taps are idempotent', (tester) async { + await tester.pumpWidget(await app()); + await settle(tester, times: 2); + await submit(tester); + await settle(tester); + parser.completeWith(_query, _onlineQuery); + await settle(tester, by: const Duration(milliseconds: 40), times: 5); + + // Two taps in the same frame, before any rebuild can remove the button. + final button = find.text('Show latest'); + await tester.tap(button, warnIfMissed: false); + await tester.tap(button, warnIfMissed: false); + await settle(tester, by: const Duration(milliseconds: 40), times: 5); + + expect(find.text('HQ Authoritative Rally 2026'), findsOneWidget); + expect(find.textContaining('HQ has fresh results'), findsNothing); + expect(tester.takeException(), isNull); + }); + + testWidgets('6. online failing after the fallback keeps the local result', + (tester) async { + await tester.pumpWidget(await app()); + await settle(tester, times: 2); + await submit(tester); + await settle(tester); + + parser.failWith(_query, StateError('backend down')); + await settle(tester, by: const Duration(milliseconds: 40), times: 5); + + expect(find.text('Rally Alpha 2025'), findsOneWidget); + expect(find.textContaining("can't reach HQ"), findsOneWidget); + expect(find.textContaining('HQ has fresh results'), findsNothing); + }); + + testWidgets('7. known offline answers locally without waiting for the budget', + (tester) async { + await tester.pumpWidget(await app(online: false)); + await settle(tester, times: 2); + await submit(tester); + await settle(tester, by: const Duration(milliseconds: 20), times: 5); + + expect(find.text('Rally Alpha 2025'), findsOneWidget); + // The online path was never entered. + expect(parser.calls, 0); + expect(telemetry.records.single.resultSource, SearchResultSource.offline); + }); + + testWidgets( + '8. a query the local parser cannot answer produces no local fallback', + (tester) async { + await tester.pumpWidget(await app()); + await settle(tester, times: 2); + await submit(tester, 'rallies in norwhere'); + await settle(tester, times: 8); + + // Past the budget, still waiting for the authoritative answer. + expect(find.textContaining('Taking the service road'), findsNothing); + expect(find.byType(CircularProgressIndicator), findsWidgets); + + parser.completeWith('rallies in norwhere', _onlineQuery); + await settle(tester, by: const Duration(milliseconds: 40), times: 5); + expect(find.text('HQ Authoritative Rally 2026'), findsOneWidget); + }); + + testWidgets('9. a clarification from the backend is preserved, not replaced', + (tester) async { + await tester.pumpWidget(await app(budget: const Duration(seconds: 2))); + await settle(tester, times: 2); + await submit(tester); + + parser.clarify(_query, 'Which rally do you mean?'); + await settle(tester, by: const Duration(milliseconds: 40), times: 5); + + expect(find.text('Which rally do you mean?'), findsOneWidget); + expect(find.text('Rally Alpha 2025'), findsNothing); + }); + + testWidgets('11. a missing local snapshot degrades to the sync prompt', + (tester) async { + // Opening a file-backed database is real I/O, so it runs outside the + // test's fake-async zone. + final widget = (await tester.runAsync( + () => app(online: false, withSnapshot: false), + ))!; + await tester.pumpWidget(widget); + await settle(tester, times: 2); + await submit(tester); + await settle(tester); + + expect(find.textContaining("haven't packed the service notes"), findsOneWidget); + expect(find.text('Sync now'), findsOneWidget); + expect(tester.takeException(), isNull); + }); + + testWidgets('12. leaving the screen mid-request causes no post-dispose update', + (tester) async { + await tester.pumpWidget(await app()); + await settle(tester, times: 2); + await submit(tester); + await settle(tester); + + // Tear the screen down while the online request is still in flight. + await tester.pumpWidget(const MaterialApp(home: SizedBox.shrink())); + parser.completeWith(_query, _onlineQuery); + await settle(tester, by: const Duration(milliseconds: 60), times: 8); + + expect(tester.takeException(), isNull); + }); + + testWidgets('13. a late result for query A cannot overwrite query B', + (tester) async { + await tester.pumpWidget(await app(budget: const Duration(seconds: 3))); + await settle(tester, times: 2); + + await submit(tester, 'rallies in ireland'); + await tester.pump(const Duration(milliseconds: 50)); + await submit(tester, 'rallies in portugal'); + await tester.pump(const Duration(milliseconds: 50)); + + // B answers first. + parser.completeWith( + 'rallies in portugal', + const SearchQuery(intent: SearchIntent.searchRallies, countries: ['Portugal']), + ); + await settle(tester, by: const Duration(milliseconds: 40), times: 5); + expect(find.text('HQ Authoritative Rally 2026'), findsOneWidget); + final afterB = tester.widget(find.byType(TextField).first); + expect(afterB.controller!.text, 'rallies in portugal'); + + // A answers late; it belongs to a superseded generation and is dropped. + parser.completeWith('rallies in ireland', _onlineQuery); + await settle(tester, by: const Duration(milliseconds: 40), times: 6); + + expect(find.textContaining('HQ has fresh results'), findsNothing); + expect(find.textContaining('Taking the service road'), findsNothing); + final afterA = tester.widget(find.byType(TextField).first); + expect(afterA.controller!.text, 'rallies in portugal'); + expect(tester.takeException(), isNull); + }); + + testWidgets( + '14. local and online landing together resolves deterministically to online', + (tester) async { + for (var i = 0; i < 5; i++) { + parser = _ControlledParser(); + telemetry = InMemorySearchTelemetrySink(); + // A distinct key per iteration forces a fresh State, so the screen picks + // up this iteration's parser instead of reusing the first one. + await tester.pumpWidget( + await app(budget: const Duration(seconds: 1), key: ValueKey(i)), + ); + await settle(tester, times: 2); + await submit(tester); + // Complete online in the same turn the local search resolves in. + parser.completeWith(_query, _onlineQuery); + await settle(tester, by: const Duration(milliseconds: 30), times: 6); + + expect(find.text('HQ Authoritative Rally 2026'), findsOneWidget, + reason: 'iteration $i'); + expect(find.textContaining('Taking the service road'), findsNothing, + reason: 'iteration $i'); + } + }); +} diff --git a/test/latency/search_latency_screen_test.dart b/test/latency/search_latency_screen_test.dart new file mode 100644 index 0000000..eb97ee6 --- /dev/null +++ b/test/latency/search_latency_screen_test.dart @@ -0,0 +1,496 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:ai_rally_search/l10n/generated/app_localizations.dart'; +import 'package:ai_rally_search/screens/general_search_screen.dart'; +import 'package:ai_rally_search/services/latency/latency_policy.dart'; +import 'package:ai_rally_search/services/latency/search_latency_coordinator.dart'; +import 'package:ai_rally_search/services/latency/search_telemetry.dart'; +import 'package:ai_rally_search/services/offline/offline_database.dart'; +import 'package:ai_rally_search/services/offline/offline_search_engine.dart'; +import 'package:ai_rally_search/services/python_search_api_client.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:http/http.dart' as http; +import 'package:http/testing.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +/// The screen-level contract for the 4-second progressive fallback. +/// +/// These drive the real widget through a real [PythonSearchApiClient] backed by +/// a scripted transport, so the trace-id header, the decode path and the +/// coordinator all take part rather than being stubbed out. + +class _Probe implements ConnectivityProbe { + bool online; + _Probe(this.online); + @override + Future isOnline() async => online; +} + +Map _snapshot() => { + 'schema_version': 1, + 'data_version': 'v1', + 'snapshot_id': '1-v1-core', + 'segment': 'core', + 'generated_at': '2026-08-30T00:00:00Z', + 'rallies': >[ + { + 'event_id': 'ev1', + 'event_name': 'Rally Alpha 2025', + 'country': 'Ireland', + 'city': 'Cork', + 'year': 2025, + 'start_date': '2025-05-01', + 'end_date': null, + 'status': null, + 'stages_count': 3, + }, + ], + 'people': const [], + 'stages': const [], + 'participation': const [], + 'final_results': const [], + 'driver_wins': const [], + 'uploader_stats': const [], + 'video_meta': const [], + 'video_actions': const [], + }; + +/// A backend conversation response naming one rally, so an online render is +/// visually distinguishable from the local one. +String _onlineBody(String rallyName, {int requestId = 1}) => jsonEncode({ + 'requestId': requestId, + 'traceId': 'server-echo', + 'session': { + 'activeQuery': {'intent': 'SEARCH_RALLIES'}, + 'referents': {}, + 'history': const [], + 'inheritedFields': const [], + 'currentRefinementFields': const [], + 'activeRequestId': requestId, + }, + 'result': { + 'parsedQuery': {'intent': 'SEARCH_RALLIES'}, + 'resolvedQuery': {'intent': 'SEARCH_RALLIES'}, + 'requiresClarification': false, + 'searchResponse': { + 'intent': 'SEARCH_RALLIES', + 'results': [ + {'kind': 'rally', 'event_id': 'online-1', 'event_name': rallyName}, + ], + 'total_count': 1, + 'has_more': false, + 'limit': 20, + 'offset': 0, + }, + }, + }); + +String _clarificationBody({int requestId = 1}) => jsonEncode({ + 'requestId': requestId, + 'traceId': 'server-echo', + 'session': { + 'activeQuery': {'intent': 'SEARCH_RALLIES'}, + 'referents': {}, + 'history': const [], + 'inheritedFields': const [], + 'currentRefinementFields': const [], + 'activeRequestId': requestId, + }, + 'result': { + 'parsedQuery': {'intent': 'SEARCH_RALLIES'}, + 'requiresClarification': true, + 'clarificationQuestion': 'Which Donegal rally did you mean?', + 'candidates': const [], + }, + }); + +/// A transport whose responses are scripted per call: each entry gives a delay +/// and either a body or an error. +class _ScriptedTransport { + final List delays; + final List bodies; + final List errors; + final List seenTraceIds = []; + int calls = 0; + + _ScriptedTransport({ + required this.delays, + required this.bodies, + List? errors, + }) : errors = errors ?? List.filled(bodies.length, null); + + http.Client get client => MockClient((request) async { + final index = calls++; + seenTraceIds.add(request.headers[PythonSearchApiClient.requestIdHeader]); + await Future.delayed(delays[index]); + final error = errors[index]; + if (error != null) throw error; + return http.Response( + bodies[index]!, + 200, + headers: {'content-type': 'application/json'}, + ); + }); +} + +Future _localEngine() async { + final db = await OfflineDatabase.open( + factory: databaseFactoryFfiNoIsolate, + path: inMemoryDatabasePath, + ); + await db.importSnapshot(_snapshot()); + return OfflineSearchEngine.create(db); +} + +Future _emptyEngine() async { + // `inMemoryDatabasePath` is shared within an isolate, so a genuinely empty + // store needs its own file. + // Created and cleaned up synchronously: real async file I/O inside a widget + // test's fake-async zone never completes. + final dir = Directory.systemTemp.createTempSync('rally_latency_empty'); + addTearDown(() => dir.deleteSync(recursive: true)); + final db = await OfflineDatabase.open( + factory: databaseFactoryFfiNoIsolate, + path: '${dir.path}/empty.db', + ); + return OfflineSearchEngine.create(db); +} + +const _policy = LatencyPolicy( + onlineResultBudget: Duration(milliseconds: 4000), + overallOnlineTimeout: Duration(seconds: 20), +); + +Widget _app({ + required OfflineSearchEngine engine, + required http.Client transport, + bool online = true, + SearchTelemetrySink? sink, + LatencyPolicy policy = _policy, +}) { + return MaterialApp( + localizationsDelegates: AppLocalizations.localizationsDelegates, + supportedLocales: AppLocalizations.supportedLocales, + home: GeneralSearchScreen( + offlineEngine: engine, + connectivityProbe: _Probe(online), + latencyPolicy: policy, + telemetrySink: sink, + pythonApiClient: PythonSearchApiClient( + baseUrl: Uri.parse('https://backend.test/'), + httpClient: transport, + policy: policy, + ), + ), + ); +} + +Future _submit(WidgetTester tester, String query) async { + await tester.enterText(find.byType(TextField).first, query); + await tester.testTextInput.receiveAction(TextInputAction.search); + await tester.pump(); +} + +/// Advances the fake clock in small steps so pending timers and the sqflite +/// microtask work both get a chance to run. +Future _settle(WidgetTester tester, Duration total) async { + const step = Duration(milliseconds: 100); + for (var elapsed = Duration.zero; elapsed < total; elapsed += step) { + await tester.pump(step); + } +} + +void main() { + setUpAll(sqfliteFfiInit); + + testWidgets('online answering inside the budget renders the online result', + (tester) async { + final sink = InMemorySearchTelemetrySink(); + final transport = _ScriptedTransport( + delays: [const Duration(milliseconds: 500)], + bodies: [_onlineBody('HQ Rally')], + ); + await tester.pumpWidget(await _localEngine().then( + (e) => _app(engine: e, transport: transport.client, sink: sink))); + await tester.pump(); + + await _submit(tester, 'rallies in ireland in 2025'); + await _settle(tester, const Duration(seconds: 2)); + + expect(find.text('HQ Rally'), findsOneWidget); + expect(find.byKey(const Key('freshResultsBanner')), findsNothing); + expect(sink.records.single.resultSource, SearchResultSource.online); + expect(sink.records.single.fallbackTriggered, isFalse); + }); + + testWidgets('online completing just before 4s does not trigger a fallback', + (tester) async { + final sink = InMemorySearchTelemetrySink(); + final transport = _ScriptedTransport( + delays: [const Duration(milliseconds: 3800)], + bodies: [_onlineBody('HQ Rally')], + ); + await tester.pumpWidget(await _localEngine().then( + (e) => _app(engine: e, transport: transport.client, sink: sink))); + await tester.pump(); + + await _submit(tester, 'rallies in ireland in 2025'); + await _settle(tester, const Duration(milliseconds: 3500)); + expect(find.text('HQ Rally'), findsNothing, reason: 'still in flight'); + + await _settle(tester, const Duration(seconds: 1)); + expect(find.text('HQ Rally'), findsOneWidget); + expect(sink.records.single.fallbackTriggered, isFalse); + expect(sink.records.single.resultSource, SearchResultSource.online); + }); + + testWidgets( + 'online exceeding 4s shows saved data, then offers the late result', + (tester) async { + final sink = InMemorySearchTelemetrySink(); + final transport = _ScriptedTransport( + delays: [const Duration(seconds: 8)], + bodies: [_onlineBody('HQ Rally')], + ); + await tester.pumpWidget(await _localEngine().then( + (e) => _app(engine: e, transport: transport.client, sink: sink))); + await tester.pump(); + + await _submit(tester, 'rallies in ireland in 2025'); + await _settle(tester, const Duration(milliseconds: 4600)); + + // The local answer is on screen and labelled as saved data. + expect(find.text('Rally Alpha 2025'), findsOneWidget); + expect(find.text('Taking the service road'), findsOneWidget); + expect(sink.records.single.resultSource, SearchResultSource.offlineFallback); + expect(sink.records.single.fallbackTriggered, isTrue); + expect(sink.records.single.fallbackTriggerMs, greaterThanOrEqualTo(4000)); + + // The authoritative result lands later and is offered, never applied. + await _settle(tester, const Duration(seconds: 5)); + expect(find.byKey(const Key('freshResultsBanner')), findsOneWidget); + expect(find.text('HQ has fresh results'), findsOneWidget); + expect(find.text('Rally Alpha 2025'), findsOneWidget, + reason: 'saved data must not be swapped out on its own'); + expect(find.text('HQ Rally'), findsNothing); + }); + + testWidgets('tapping "Show latest" promotes the authoritative result', + (tester) async { + final transport = _ScriptedTransport( + delays: [const Duration(seconds: 8)], + bodies: [_onlineBody('HQ Rally')], + ); + await tester.pumpWidget(await _localEngine() + .then((e) => _app(engine: e, transport: transport.client))); + await tester.pump(); + + await _submit(tester, 'rallies in ireland in 2025'); + await _settle(tester, const Duration(seconds: 10)); + expect(find.text('Show latest'), findsOneWidget); + + await tester.tap(find.text('Show latest')); + await _settle(tester, const Duration(milliseconds: 600)); + + expect(find.text('HQ Rally'), findsOneWidget); + expect(find.text('Rally Alpha 2025'), findsNothing); + expect(find.byKey(const Key('freshResultsBanner')), findsNothing); + }); + + testWidgets('repeated "Show latest" taps are idempotent', (tester) async { + final transport = _ScriptedTransport( + delays: [const Duration(seconds: 8)], + bodies: [_onlineBody('HQ Rally')], + ); + await tester.pumpWidget(await _localEngine() + .then((e) => _app(engine: e, transport: transport.client))); + await tester.pump(); + + await _submit(tester, 'rallies in ireland in 2025'); + await _settle(tester, const Duration(seconds: 10)); + + // Two taps in the same frame: the second finds nothing pending. + await tester.tap(find.text('Show latest'), warnIfMissed: false); + await tester.tap(find.text('Show latest'), warnIfMissed: false); + await _settle(tester, const Duration(milliseconds: 600)); + + expect(find.text('HQ Rally'), findsOneWidget); + expect(tester.takeException(), isNull); + }); + + testWidgets('online failing after a fallback leaves the saved data in place', + (tester) async { + final transport = _ScriptedTransport( + delays: [const Duration(seconds: 8)], + bodies: [null], + errors: [http.ClientException('connection reset')], + ); + await tester.pumpWidget(await _localEngine() + .then((e) => _app(engine: e, transport: transport.client))); + await tester.pump(); + + await _submit(tester, 'rallies in ireland in 2025'); + await _settle(tester, const Duration(seconds: 10)); + + expect(find.text('Rally Alpha 2025'), findsOneWidget); + expect(find.byKey(const Key('freshResultsBanner')), findsNothing); + expect(find.text("The pit crew can't reach HQ right now"), findsOneWidget); + }); + + testWidgets('a query the local parser cannot answer waits for the backend', + (tester) async { + final sink = InMemorySearchTelemetrySink(); + final transport = _ScriptedTransport( + delays: [const Duration(seconds: 6)], + bodies: [_onlineBody('HQ Rally')], + ); + await tester.pumpWidget(await _localEngine().then( + (e) => _app(engine: e, transport: transport.client, sink: sink))); + await tester.pump(); + + await _submit(tester, 'rallies in nowhereatall'); + await _settle(tester, const Duration(milliseconds: 5000)); + // Past the budget, but nothing local was fabricated. + expect(find.text('Taking the service road'), findsNothing); + expect(sink.records, isEmpty); + + await _settle(tester, const Duration(seconds: 2)); + expect(find.text('HQ Rally'), findsOneWidget); + expect(sink.records.single.resultSource, SearchResultSource.online); + expect(sink.records.single.localParserCouldAnswer, isFalse); + }); + + testWidgets('a slow clarification is still delivered as a clarification', + (tester) async { + final transport = _ScriptedTransport( + delays: [const Duration(seconds: 6)], + bodies: [_clarificationBody()], + ); + // A query with no safe local answer, so the clarification is not pre-empted + // by a fallback. + await tester.pumpWidget(await _localEngine() + .then((e) => _app(engine: e, transport: transport.client))); + await tester.pump(); + + await _submit(tester, 'rallies in nowhereatall'); + await _settle(tester, const Duration(seconds: 8)); + + expect(find.text('Which Donegal rally did you mean?'), findsOneWidget); + }); + + testWidgets('a late response to query A cannot overwrite query B', + (tester) async { + final transport = _ScriptedTransport( + delays: [const Duration(seconds: 6), const Duration(milliseconds: 300)], + bodies: [ + _onlineBody('STALE Rally A', requestId: 1), + _onlineBody('FRESH Rally B', requestId: 2), + ], + ); + await tester.pumpWidget(await _localEngine() + .then((e) => _app(engine: e, transport: transport.client))); + await tester.pump(); + + await _submit(tester, 'rallies in nowhereatall'); + await tester.pump(const Duration(milliseconds: 200)); + await _submit(tester, 'rallies in stillnowhere'); + await _settle(tester, const Duration(seconds: 10)); + + expect(transport.calls, 2); + expect(find.text('FRESH Rally B'), findsOneWidget); + expect(find.text('STALE Rally A'), findsNothing); + expect(find.byKey(const Key('freshResultsBanner')), findsNothing); + }); + + testWidgets('leaving the screen mid-request does not update a disposed state', + (tester) async { + final transport = _ScriptedTransport( + delays: [const Duration(seconds: 6)], + bodies: [_onlineBody('HQ Rally')], + ); + await tester.pumpWidget(await _localEngine() + .then((e) => _app(engine: e, transport: transport.client))); + await tester.pump(); + + await _submit(tester, 'rallies in ireland in 2025'); + await tester.pump(const Duration(milliseconds: 200)); + + // Replace the screen while the request is still in flight. + await tester.pumpWidget(const MaterialApp(home: SizedBox.shrink())); + await _settle(tester, const Duration(seconds: 10)); + + expect(tester.takeException(), isNull); + }); + + testWidgets('known offline answers from the snapshot without any request', + (tester) async { + final transport = _ScriptedTransport( + delays: [Duration.zero], + bodies: [_onlineBody('HQ Rally')], + ); + final sink = InMemorySearchTelemetrySink(); + await tester.pumpWidget(await _localEngine().then((e) => _app( + engine: e, + transport: transport.client, + online: false, + sink: sink, + ))); + await tester.pump(); + + await _submit(tester, 'rallies in ireland in 2025'); + await _settle(tester, const Duration(seconds: 1)); + + expect(transport.calls, 0); + expect(find.text('Rally Alpha 2025'), findsOneWidget); + expect(sink.records.single.resultSource, SearchResultSource.offline); + expect(sink.records.single.connectivity, ConnectivityState.offline); + }); + + testWidgets('offline with no snapshot offers a sync instead of a result', + (tester) async { + final transport = _ScriptedTransport( + delays: [Duration.zero], + bodies: [_onlineBody('HQ Rally')], + ); + // Opening a file-backed database is real I/O, which never completes inside + // the test's fake-async zone — so it runs outside it. + final engine = (await tester.runAsync(_emptyEngine))!; + await tester.pumpWidget(_app( + engine: engine, + transport: transport.client, + online: false, + )); + await tester.pump(); + + await _submit(tester, 'rallies in ireland in 2025'); + await _settle(tester, const Duration(seconds: 1)); + + expect(find.text("We haven't packed the service notes yet"), findsOneWidget); + expect(transport.calls, 0); + }); + + testWidgets('the correlation id reaches the backend as X-Request-Id', + (tester) async { + final sink = InMemorySearchTelemetrySink(); + final transport = _ScriptedTransport( + delays: [const Duration(milliseconds: 300)], + bodies: [_onlineBody('HQ Rally')], + ); + await tester.pumpWidget(await _localEngine().then( + (e) => _app(engine: e, transport: transport.client, sink: sink))); + await tester.pump(); + + await _submit(tester, 'rallies in ireland in 2025'); + await _settle(tester, const Duration(seconds: 2)); + + final sent = transport.seenTraceIds.single; + expect(sent, isNotNull); + expect(sent, matches(RegExp(r'^[0-9a-f]{32}$'))); + // The same id is what the client-side latency record is keyed by, so a + // client record and a backend timing line join without either logging the + // query text. + expect(sink.records.single.requestId, sent); + }); +} diff --git a/test/models/rally_stream_test.dart b/test/models/rally_stream_test.dart deleted file mode 100644 index a6d71ad..0000000 --- a/test/models/rally_stream_test.dart +++ /dev/null @@ -1,49 +0,0 @@ -import 'package:flutter_test/flutter_test.dart'; -import 'package:ai_rally_search/models/rally_stream.dart'; - -void main() { - group('RallyStream Model Tests', () { - test('parses from database map with clipStartTime and clipDuration', () { - final map = { - 'id': 85989, - 'video_id': 181564, - 'clip_duration': 7.17, - 'video_type': 'sendObs', - 'on_demand_url': 'https://stream.example.com/manifest.m3u8', - 'created_at': '2026-03-01 14:30:00', - 'updated_at': '2026-03-01 14:35:00', - 'clip_start_time': 7496.678, - 'clip_status': 'complete', - 'download_counter': 12, - 'share_counter': 5, - }; - - final stream = RallyStream.fromMap(map); - - expect(stream.id, 85989); - expect(stream.videoId, 181564); - expect(stream.clipDuration, closeTo(7.17, 0.001)); - expect(stream.clipStartTime, closeTo(7496.678, 0.001)); - expect(stream.videoType, 'sendObs'); - expect(stream.clipStatus, 'complete'); - expect(stream.formattedDuration, '7.2s'); - expect(stream.formattedClipRange, '2:04:56 → 2:05:03'); - }); - - test('handles formattedClipRange when clipStartTime is 0 or clipDuration is not specified', () { - final stream1 = RallyStream( - id: 1, - clipStartTime: 0.0, - clipDuration: 30.0, - ); - expect(stream1.formattedClipRange, '00:00 → 00:30'); - - final stream2 = RallyStream( - id: 2, - clipStartTime: 120.0, - clipDuration: null, - ); - expect(stream2.formattedClipRange, '02:00 → End'); - }); - }); -} diff --git a/test/offline/offline_benchmark_test.dart b/test/offline/offline_benchmark_test.dart index daa5de3..38ef4fa 100644 --- a/test/offline/offline_benchmark_test.dart +++ b/test/offline/offline_benchmark_test.dart @@ -6,7 +6,9 @@ import 'package:ai_rally_search/models/search_query.dart'; import 'package:ai_rally_search/services/offline/offline_database.dart'; import 'package:ai_rally_search/services/offline/offline_query_parser.dart'; import 'package:ai_rally_search/services/offline/offline_search_engine.dart'; -import 'package:ai_rally_search/services/offline/offline_search_router.dart'; +import 'package:ai_rally_search/services/latency/latency_policy.dart'; +import 'package:ai_rally_search/services/latency/search_latency_coordinator.dart'; +import 'package:ai_rally_search/services/latency/search_telemetry.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:sqflite_common_ffi/sqflite_ffi.dart'; @@ -395,10 +397,34 @@ class _Probe implements ConnectivityProbe { Future>> _connectivityScenarios(OfflineSearchEngine engine) async { final results = >[]; - Future run(String scenario, ConnectivityProbe probe, Future Function() online, {Duration budget = const Duration(seconds: 4)}) async { - final router = OfflineSearchRouter(connectivity: probe, engine: engine, fallbackBudget: budget); - final r = await router.route(rawText: 'rallies in ireland in 2025', online: online); - results.add({'scenario': scenario, 'mode': r.mode.name, 'ux_state': r.uxState.name, 'silent_swap': false}); + Future run( + String scenario, + ConnectivityProbe probe, + Future Function() online, { + Duration budget = const Duration(seconds: 4), + }) async { + final coordinator = SearchLatencyCoordinator( + connectivity: probe, + engine: engine, + policy: LatencyPolicy( + onlineResultBudget: budget, + overallOnlineTimeout: const Duration(seconds: 10), + ), + ); + final events = await coordinator + .run(generation: 1, rawText: 'rallies in ireland in 2025', online: online) + .toList(); + // A late authoritative result is offered, never applied, so no scenario can + // silently swap what the user is looking at. + final swapped = events + .where((e) => e.stage == SearchStage.lateOnlineAvailable) + .any((e) => e.isTerminalRender); + results.add({ + 'scenario': scenario, + 'stages': events.map((e) => e.stage.name).toList(), + 'result_source': events.first.source.wireName, + 'silent_swap': swapped, + }); } await run('ONLINE', _Probe(true), () async => 'ONLINE'); diff --git a/test/offline/offline_router_sync_messaging_test.dart b/test/offline/offline_router_sync_messaging_test.dart index 04aeeb0..b13471b 100644 --- a/test/offline/offline_router_sync_messaging_test.dart +++ b/test/offline/offline_router_sync_messaging_test.dart @@ -1,8 +1,12 @@ +import 'dart:io'; + import 'package:ai_rally_search/services/offline/offline_database.dart'; import 'package:ai_rally_search/services/offline/offline_messaging.dart'; import 'package:ai_rally_search/services/offline/offline_search_engine.dart'; -import 'package:ai_rally_search/services/offline/offline_search_router.dart'; +import 'package:ai_rally_search/services/latency/latency_policy.dart'; +import 'package:ai_rally_search/services/latency/search_latency_coordinator.dart'; +import 'package:ai_rally_search/services/latency/search_telemetry.dart'; import 'package:ai_rally_search/services/offline/offline_snapshot_sync.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:sqflite_common_ffi/sqflite_ffi.dart'; @@ -66,50 +70,228 @@ void main() { }); }); - group('NETWORK_FIRST_WITH_LOCAL_FALLBACK router', () { - test('ONLINE -> authoritative online result is used', () async { - final router = OfflineSearchRouter(connectivity: _FakeProbe(true), engine: await engine()); - final r = await router.route(rawText: 'rallies in ireland', online: () async => 'ONLINE'); - expect(r.mode, RouteMode.onlineAuthoritative); - expect(r.online, 'ONLINE'); + group('latency coordinator (single fallback policy)', () { + SearchLatencyCoordinator coordinator( + OfflineSearchEngine e, { + bool online = true, + Duration budget = const Duration(milliseconds: 50), + Duration overall = const Duration(seconds: 5), + }) => + SearchLatencyCoordinator( + connectivity: _FakeProbe(online), + engine: e, + policy: LatencyPolicy( + onlineResultBudget: budget, + overallOnlineTimeout: overall, + ), + ); + + Future>> collect(Stream> s) => + s.toList(); + + test('online within budget -> authoritative online result', () async { + final events = await collect(coordinator(await engine()).run( + generation: 1, + rawText: 'rallies in ireland', + online: () async => 'ONLINE', + )); + expect(events.single.stage, SearchStage.online); + expect(events.single.online, 'ONLINE'); + expect(events.single.source, SearchResultSource.online); }); - test('OFFLINE -> local search used immediately (online skipped)', () async { + test('known offline -> local immediately, online never attempted', () async { var onlineCalled = false; - final router = OfflineSearchRouter(connectivity: _FakeProbe(false), engine: await engine()); - final r = await router.route(rawText: 'rallies in ireland', online: () async { - onlineCalled = true; - return 'ONLINE'; - }); + final events = await collect( + coordinator(await engine(), online: false).run( + generation: 1, + rawText: 'rallies in ireland', + online: () async { + onlineCalled = true; + return 'ONLINE'; + }, + ), + ); expect(onlineCalled, isFalse); - expect(r.mode, RouteMode.offlineLocal); - expect(r.offline!.hasResults, isTrue); + expect(events.single.stage, SearchStage.offlineImmediate); + expect(events.single.offline!.hasResults, isTrue); }); - test('TIMEOUT -> local fallback surfaced, online kept for explicit promotion (no silent swap)', () async { - final router = OfflineSearchRouter( - connectivity: _FakeProbe(true), - engine: await engine(), - fallbackBudget: const Duration(milliseconds: 50), - ); - final r = await router.route( + test('budget exceeded -> local shown, then late online is offered not applied', + () async { + final events = await collect(coordinator(await engine()).run( + generation: 7, rawText: 'rallies in ireland', - online: () => Future.delayed(const Duration(seconds: 2), () => 'ONLINE'), - ); - expect(r.mode, RouteMode.lowBandwidthLocal); - expect(r.offline, isNotNull); - expect(r.pendingOnline, isNotNull); // caller decides when/if to promote - expect(r.uxState, OfflineUxState.lowBandwidthLocalFallback); + online: () => + Future.delayed(const Duration(milliseconds: 300), () => 'ONLINE'), + )); + expect(events.map((e) => e.stage), [ + SearchStage.offlineFallback, + SearchStage.lateOnlineAvailable, + ]); + expect(events.first.source, SearchResultSource.offlineFallback); + // The late result is explicitly not a render: it must be offered. + expect(events.last.isTerminalRender, isFalse); + expect(events.last.online, 'ONLINE'); + expect(events.every((e) => e.generation == 7), isTrue); + }); + + test('online fails after fallback -> local stays, no error render', () async { + final events = await collect(coordinator(await engine()).run( + generation: 1, + rawText: 'rallies in ireland', + online: () => Future.delayed( + const Duration(milliseconds: 300), () => throw StateError('boom')), + )); + expect(events.map((e) => e.stage), [ + SearchStage.offlineFallback, + SearchStage.lateOnlineFailed, + ]); + expect(events.last.isTerminalRender, isFalse); }); - test('BACKEND ERROR -> deterministic local fallback', () async { - final router = OfflineSearchRouter(connectivity: _FakeProbe(true), engine: await engine()); - final r = await router.route( + test('online fails before budget -> deterministic local fallback', () async { + final events = await collect(coordinator(await engine()).run( + generation: 1, rawText: 'rallies in ireland', online: () async => throw StateError('boom'), + )); + expect(events.single.stage, SearchStage.offlineAfterOnlineFailure); + expect(events.single.offline!.hasResults, isTrue); + }); + + test('no safe local answer -> waits for online rather than fabricating one', + () async { + final events = await collect(coordinator(await engine()).run( + generation: 1, + // Nothing in the snapshot matches, so the local parser cannot answer. + rawText: 'rallies in absolutelynowhere', + online: () => + Future.delayed(const Duration(milliseconds: 300), () => 'ONLINE'), + )); + expect(events.single.stage, SearchStage.online); + expect(events.single.online, 'ONLINE'); + }); + + test('no safe local answer and online fails -> surfaces the failure', + () async { + final events = await collect(coordinator(await engine()).run( + generation: 1, + rawText: 'rallies in absolutelynowhere', + online: () => Future.delayed( + const Duration(milliseconds: 200), () => throw StateError('boom')), + )); + expect(events.single.stage, SearchStage.onlineFailed); + }); + + test('online just inside the budget wins; just outside it falls back', + () async { + final inside = await collect(coordinator( + await engine(), + budget: const Duration(milliseconds: 300), + ).run( + generation: 1, + rawText: 'rallies in ireland', + online: () => + Future.delayed(const Duration(milliseconds: 60), () => 'ONLINE'), + )); + expect(inside.single.stage, SearchStage.online); + + final outside = await collect(coordinator( + await engine(), + budget: const Duration(milliseconds: 60), + ).run( + generation: 1, + rawText: 'rallies in ireland', + online: () => + Future.delayed(const Duration(milliseconds: 400), () => 'ONLINE'), + )); + expect(outside.first.stage, SearchStage.offlineFallback); + expect(outside.last.stage, SearchStage.lateOnlineAvailable); + }); + + test('local and online completing together resolves to online', () async { + // Both are ready effectively at once; the rule is that an online answer + // inside the budget always wins, so the outcome is never a coin flip. + for (var i = 0; i < 12; i++) { + final events = await collect(coordinator( + await engine(), + budget: const Duration(milliseconds: 200), + ).run( + generation: 1, + rawText: 'rallies in ireland', + online: () async => 'ONLINE', + )); + expect(events.single.stage, SearchStage.online, reason: 'iteration $i'); + } + }); + + // `inMemoryDatabasePath` is shared within an isolate, so an "empty" local + // store needs its own file to genuinely have no snapshot. + Future emptyEngine() async { + final dir = await Directory.systemTemp.createTemp('rally_empty_snapshot'); + final db = await OfflineDatabase.open( + factory: databaseFactoryFfi, + path: '${dir.path}/empty.db', + ); + addTearDown(() async => dir.delete(recursive: true)); + return OfflineSearchEngine.create(db); + } + + test('a stale/missing snapshot never fabricates a local answer', () async { + final empty = await emptyEngine(); + expect(await empty.database.hasSnapshot(), isFalse); + final events = await collect( + SearchLatencyCoordinator( + connectivity: _FakeProbe(true), + engine: empty, + policy: const LatencyPolicy( + onlineResultBudget: Duration(milliseconds: 40), + overallOnlineTimeout: Duration(seconds: 5), + ), + ).run( + generation: 1, + rawText: 'rallies in ireland', + online: () => + Future.delayed(const Duration(milliseconds: 200), () => 'ONLINE'), + ), + ); + expect(events.single.stage, SearchStage.online); + }); + + test('offline device with no snapshot reports failure, not a fake result', + () async { + final events = await collect( + SearchLatencyCoordinator( + connectivity: _FakeProbe(false), + engine: await emptyEngine(), + policy: const LatencyPolicy(), + ).run( + generation: 1, + rawText: 'rallies in ireland', + online: () async => 'ONLINE', + ), + ); + expect(events.single.stage, SearchStage.onlineFailed); + }); + + test('no offline stack at all falls through to the online path', () async { + final events = await collect( + SearchLatencyCoordinator( + connectivity: null, + engine: null, + policy: const LatencyPolicy( + onlineResultBudget: Duration(milliseconds: 30), + overallOnlineTimeout: Duration(seconds: 5), + ), + ).run( + generation: 1, + rawText: 'rallies in ireland', + online: () => + Future.delayed(const Duration(milliseconds: 150), () => 'ONLINE'), + ), ); - expect(r.mode, RouteMode.backendUnreachableLocal); - expect(r.offline!.hasResults, isTrue); + expect(events.single.stage, SearchStage.online); }); }); diff --git a/test/screens/offline_search_screen_test.dart b/test/screens/offline_search_screen_test.dart index a2e9a2a..a30a918 100644 --- a/test/screens/offline_search_screen_test.dart +++ b/test/screens/offline_search_screen_test.dart @@ -2,7 +2,7 @@ import 'package:ai_rally_search/l10n/generated/app_localizations.dart'; import 'package:ai_rally_search/screens/general_search_screen.dart'; import 'package:ai_rally_search/services/offline/offline_database.dart'; import 'package:ai_rally_search/services/offline/offline_search_engine.dart'; -import 'package:ai_rally_search/services/offline/offline_search_router.dart'; +import 'package:ai_rally_search/services/latency/search_latency_coordinator.dart'; import 'package:ai_rally_search/widgets/offline_banner.dart'; import 'package:flutter/material.dart'; import 'package:flutter_test/flutter_test.dart'; diff --git a/test/screens/search_home_phase1_test.dart b/test/screens/search_home_phase1_test.dart index 1fc15bc..58295b5 100644 --- a/test/screens/search_home_phase1_test.dart +++ b/test/screens/search_home_phase1_test.dart @@ -239,7 +239,7 @@ void main() { expect(find.textContaining('transcript'), findsNothing); }); - testWidgets('browse/streams affordance remains reachable from search', ( + testWidgets('direct-DB browse/streams affordance is removed', ( tester, ) async { final repo = CountingSearchRepository(); @@ -253,9 +253,10 @@ void main() { ); await tester.pumpAndSettle(); - // The Browse entry point exists in the app bar (navigation target is the - // RallyStreamsPage, which is retained, not deleted). - expect(find.byIcon(Icons.video_library_outlined), findsOneWidget); + // The old "Browse streams" entry point navigated to RallyStreamsPage, + // which connected the device directly to RDS/MySQL. That screen and its + // direct-DB access have been removed; the affordance must be gone. + expect(find.byIcon(Icons.video_library_outlined), findsNothing); }); }); } diff --git a/test/screens/video_action_search_screen_test.dart b/test/screens/video_action_search_screen_test.dart deleted file mode 100644 index 1f7ee71..0000000 --- a/test/screens/video_action_search_screen_test.dart +++ /dev/null @@ -1,46 +0,0 @@ -import 'package:flutter/material.dart'; -import 'package:flutter_test/flutter_test.dart'; -import 'package:ai_rally_search/models/video_action_search_query.dart'; -import 'package:ai_rally_search/screens/video_action_search_screen.dart'; - -void main() { - testWidgets('VideoActionSearchScreen UI structure and filter controls render correctly', (tester) async { - await tester.pumpWidget( - const MaterialApp( - home: VideoActionSearchScreen( - initialQuery: VideoActionSearchQuery( - actionType: 'jump', - country: 'Austria', - eventName: 'OBM Land', - ), - ), - ), - ); - - // Verify AppBar - expect(find.text('Action Moments Search'), findsOneWidget); - - // Verify Search text field - expect(find.byType(TextField), findsWidgets); - expect(find.text('OBM Land'), findsOneWidget); - - // Verify Action dropdown value - expect(find.text('Jump'), findsOneWidget); - - // Verify Country dropdown label - expect(find.text('Austria (AT)'), findsOneWidget); - - // Verify Search button - expect(find.text('Search'), findsOneWidget); - - // Toggle advanced filters - final stageFiltersButton = find.text('Stage Filters'); - expect(stageFiltersButton, findsOneWidget); - await tester.tap(stageFiltersButton); - await tester.pumpAndSettle(); - - expect(find.text('Less Filters'), findsOneWidget); - expect(find.text('Stage Name (e.g. Gale Rigg)'), findsOneWidget); - expect(find.text('Stage # (e.g. 3)'), findsOneWidget); - }); -} diff --git a/test/services/deterministic_search_test.dart b/test/services/deterministic_search_test.dart deleted file mode 100644 index 23c8d2c..0000000 --- a/test/services/deterministic_search_test.dart +++ /dev/null @@ -1,194 +0,0 @@ -import 'package:flutter_test/flutter_test.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:ai_rally_search/models/video_action.dart'; -import 'package:ai_rally_search/models/video_action_search_query.dart'; -import 'package:ai_rally_search/services/database_service.dart'; -import 'package:ai_rally_search/services/video_action_repository.dart'; - -void main() { - TestWidgetsFlutterBinding.ensureInitialized(); - - late DatabaseService dbService; - late VideoActionRepository repository; - - setUpAll(() async { - await dotenv.load(fileName: '.env'); - dbService = DatabaseService(); - repository = VideoActionRepository(dbService: dbService); - }); - - tearDownAll(() async { - await dbService.close(); - }); - - group('Deterministic Video Action Search Integration Tests', () { - // 1. Search by action - test('1. Search by action: actionType = "jump"', () async { - const query = VideoActionSearchQuery(actionType: 'jump', limit: 10); - final results = await repository.searchVideoActions(query); - - expect(results, isNotEmpty); - for (final action in results) { - expect(action.actionType, equals('jump')); - } - }); - - // 2. Search by country - test('2. Search by country: country = "at" and "Austria"', () async { - const queryCode = VideoActionSearchQuery(country: 'at', limit: 10); - final resultsCode = await repository.searchVideoActions(queryCode); - - const queryName = VideoActionSearchQuery(country: 'Austria', limit: 10); - final resultsName = await repository.searchVideoActions(queryName); - - expect(resultsCode.length, equals(resultsName.length)); - if (resultsCode.isNotEmpty) { - for (final action in resultsCode) { - expect( - action.eventCountry?.toLowerCase(), - anyOf(equals('at'), contains('austria')), - ); - } - } - }); - - // 3. Search by action + country - test('3. Search by action + country: jump + United Kingdom / gb', () async { - const query = VideoActionSearchQuery( - actionType: 'jump', - country: 'United Kingdom', - limit: 10, - ); - final results = await repository.searchVideoActions(query); - - for (final action in results) { - expect(action.actionType, equals('jump')); - expect( - action.eventCountry?.toLowerCase(), - anyOf(equals('gb'), contains('united kingdom'), contains('scotland'), contains('wales')), - ); - } - }); - - // 4. Search by event name (case-insensitive substring) - test('4. Search by event name: partial "Trackrod"', () async { - const query = VideoActionSearchQuery( - eventName: 'trackrod', - limit: 10, - ); - final results = await repository.searchVideoActions(query); - - expect(results, isNotEmpty); - for (final action in results) { - expect(action.eventName?.toLowerCase(), contains('trackrod')); - } - }); - - // 5. Search by stage name and stage number - test('5. Search by stage: stageName = "Gale Rigg" and stageNumber = "3"', () async { - const query = VideoActionSearchQuery( - stageName: 'Gale Rigg', - stageNumber: '3', - limit: 10, - ); - final results = await repository.searchVideoActions(query); - - expect(results, isNotEmpty); - for (final action in results) { - expect(action.stageName?.toLowerCase(), contains('gale rigg')); - } - }); - - // 6. Multiple combined filters with AND logic - test('6. Multiple filters combined with AND logic', () async { - const query = VideoActionSearchQuery( - actionType: 'start_line', - country: 'United Kingdom', - eventName: 'Trackrod', - stageName: 'Gale Rigg', - stageNumber: '3', - limit: 10, - ); - final results = await repository.searchVideoActions(query); - - expect(results, isNotEmpty); - for (final action in results) { - expect(action.actionType, equals('start_line')); - expect(action.eventName?.toLowerCase(), contains('trackrod')); - expect(action.stageName?.toLowerCase(), contains('gale rigg')); - } - }); - - // 7. Pagination test (stable ordering, non-overlapping pages) - test('7. Pagination: limit & offset work deterministically', () async { - const queryPage1 = VideoActionSearchQuery(limit: 5, offset: 0); - const queryPage2 = VideoActionSearchQuery(limit: 5, offset: 5); - - final page1 = await repository.searchVideoActions(queryPage1); - final page2 = await repository.searchVideoActions(queryPage2); - - expect(page1.length, equals(5)); - expect(page2.length, equals(5)); - - final page1Ids = page1.map((a) => a.id).toSet(); - final page2Ids = page2.map((a) => a.id).toSet(); - - // Ensure no overlapping IDs between distinct pages - expect(page1Ids.intersection(page2Ids), isEmpty); - }); - - // 8. Empty results handling - test('8. Empty results for non-matching criteria returns empty list without error', () async { - const query = VideoActionSearchQuery( - eventName: 'NonexistentRallyEventxyz123', - ); - final results = await repository.searchVideoActions(query); - final count = await repository.countVideoActions(query); - - expect(results, isEmpty); - expect(count, equals(0)); - }); - - // 9. Unknown/invalid action type handled gracefully - test('9. Invalid/unknown action type handled gracefully', () async { - const query = VideoActionSearchQuery( - actionType: 'unknown_flying_car_segment', - ); - final results = await repository.searchVideoActions(query); - expect(results, isA>()); - expect(results, isEmpty); - }); - - // 10. Verify essential fields on all returned results - test('10. Verify all returned results contain essential playback fields', () async { - const query = VideoActionSearchQuery(limit: 20); - final results = await repository.searchVideoActions(query); - - expect(results, isNotEmpty); - for (final action in results) { - expect(action.id, isPositive); - expect(action.videoId, isPositive); - expect(action.videoUrl, isNotNull); - expect(action.videoUrl, isNotEmpty); - expect(action.actionType, isNotEmpty); - expect(action.startTime, greaterThanOrEqualTo(0.0)); - expect(action.endTime, greaterThan(action.startTime)); - expect(action.duration, greaterThan(0.0)); - } - }); - - // 11. Verify VideoAction model validity for playback - test('11. Returned VideoAction model has formatted helper getters for UI/Player', () async { - const query = VideoActionSearchQuery(actionType: 'jump', limit: 1); - final results = await repository.searchVideoActions(query); - - expect(results, isNotEmpty); - final action = results.first; - - expect(action.formattedDuration, isNotEmpty); - expect(action.formattedTimeRange, contains('→')); - expect(action.locationOrStageDescription, isNotEmpty); - expect(action.title, equals('Jump')); - }); - }); -} diff --git a/test/services/entity_search/live_in_memory_entity_search_test.dart b/test/services/entity_search/live_in_memory_entity_search_test.dart deleted file mode 100644 index b59f5ad..0000000 --- a/test/services/entity_search/live_in_memory_entity_search_test.dart +++ /dev/null @@ -1,62 +0,0 @@ -@Tags(['live-db']) -library; - -import 'dart:math'; - -import 'package:ai_rally_search/services/entity_search/entity_search_models.dart'; -import 'package:ai_rally_search/services/entity_search/in_memory_entity_search_service.dart'; -import 'package:ai_rally_search/services/entity_search/mysql_entity_search_data_source.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:flutter_test/flutter_test.dart'; - -void main() { - setUpAll(() async => dotenv.load(fileName: '.env')); - - test('live index smoke benchmark', () async { - final service = InMemoryEntitySearchService( - dataSource: MySqlEntitySearchDataSource(), - ); - final stats = await service.rebuild(); - expect(stats.entityCount, greaterThan(0)); - - final inputs = <(String, SearchEntityType)>[ - ('aluksni', SearchEntityType.rally), - ('aluksnay', SearchEntityType.rally), - ('aluksney', SearchEntityType.rally), - ('alux new', SearchEntityType.rally), - ('a looks nay', SearchEntityType.rally), - ('eluksne', SearchEntityType.rally), - ('aluknse', SearchEntityType.rally), - ('pawel malgo', SearchEntityType.person), - ('shea brain', SearchEntityType.person), - ('donny gall', SearchEntityType.rally), - ('kemel berg', SearchEntityType.stage), - ('dushniki', SearchEntityType.stage), - ]; - final micros = []; - for (final input in inputs) { - final watch = Stopwatch()..start(); - final results = await service.search( - EntitySearchRequest( - rawMention: input.$1, - entityType: input.$2, - limit: 5, - ), - ); - watch.stop(); - micros.add(watch.elapsedMicroseconds); - // ignore: avoid_print - print( - '${input.$1}: ${results.map((r) => '${r.canonicalName}=${r.score.toStringAsFixed(3)} ${r.signals.toMap()}').join(' | ')}', - ); - } - micros.sort(); - final average = micros.reduce((a, b) => a + b) / micros.length; - int percentile(double p) => - micros[min(micros.length - 1, (micros.length * p).floor())]; - // ignore: avoid_print - print( - 'INDEX entities=${stats.entityCount} buildMs=${stats.buildTime.inMicroseconds / 1000} estimatedBytes=${stats.estimatedBytes} avgUs=$average p50Us=${percentile(.50)} p95Us=${percentile(.95)} maxUs=${micros.last}', - ); - }); -} diff --git a/test/services/general_search_test.dart b/test/services/general_search_test.dart deleted file mode 100644 index 2083b4d..0000000 --- a/test/services/general_search_test.dart +++ /dev/null @@ -1,454 +0,0 @@ -// ignore_for_file: avoid_print -import 'package:flutter_test/flutter_test.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:ai_rally_search/models/search_intent.dart'; -import 'package:ai_rally_search/models/search_query.dart'; -import 'package:ai_rally_search/models/search_results.dart'; -import 'package:ai_rally_search/models/video_action.dart'; -import 'package:ai_rally_search/services/database_service.dart'; -import 'package:ai_rally_search/services/search_repository.dart'; - -void main() { - TestWidgetsFlutterBinding.ensureInitialized(); - - late DatabaseService dbService; - late SearchRepository repository; - - setUpAll(() async { - await dotenv.load(fileName: '.env'); - dbService = DatabaseService(); - repository = SearchRepository(dbService: dbService); - await dbService.connect(); - }); - - tearDownAll(() async { - await dbService.close(); - }); - - group('General Search Integration Tests across all 9 Search Intents', () { - // 1. Search rallies by country - test('1. SEARCH_RALLIES: "Rallies in Ireland" returns valid RallySearchResult list', () async { - const query = SearchQuery( - intent: SearchIntent.searchRallies, - country: 'Ireland', - ); - - final response = await repository.search(query); - expect(response.intent, equals(SearchIntent.searchRallies)); - expect(response.results, isA>()); - expect(response.totalCount, greaterThan(0)); - - final rallies = response.results.cast(); - print('\nRallies in Ireland count: ${rallies.length}'); - for (final r in rallies.take(3)) { - print(' • ${r.eventName} (${r.formattedLocation}), ${r.stagesCount} stages'); - expect(r.eventName, isNotEmpty); - expect(r.eventId, isNotEmpty); - } - }); - - // 2. Search rallies by city - test('2. SEARCH_RALLIES: "Rallies in Donegal / Letterkenny" matches specific city', () async { - const query = SearchQuery( - intent: SearchIntent.searchRallies, - city: 'Letterkenny', - ); - - final response = await repository.searchRallies(query); - expect(response.results, isNotEmpty); - for (final r in response.results) { - expect(r.city?.toLowerCase(), contains('letterkenny')); - } - }); - - // 3. Search rallies by year - test('3. SEARCH_RALLIES: "Rallies from 2025" filters events by year', () async { - const query = SearchQuery( - intent: SearchIntent.searchRallies, - year: 2025, - ); - - final response = await repository.searchRallies(query); - expect(response.results, isNotEmpty); - for (final r in response.results) { - print(' Event: ${r.eventName}, start: ${r.startDate}, end: ${r.endDate}, parsedYear: ${r.year}'); - expect( - r.year == 2025 || (r.startDate != null && r.startDate!.year == 2025) || (r.endDate != null && r.endDate!.year == 2025), - isTrue, - ); - } - }); - - - // 4. Search driver rallies (participation) - test('4. SEARCH_DRIVER_RALLIES: "Rallies Driver X participated in" returns participation records', () async { - const query = SearchQuery( - intent: SearchIntent.searchDriverRallies, - driverName: 'Josh Moffett', - ); - - final response = await repository.search(query); - expect(response.intent, equals(SearchIntent.searchDriverRallies)); - expect(response.results, isA>()); - expect(response.totalCount, greaterThan(0)); - - final participations = response.results.cast(); - print('\nJosh Moffett participations count: ${participations.length}'); - for (final p in participations) { - print(' • ${p.eventName}: ${p.driverName} - ${p.finishPositionDisplay}'); - expect(p.driverName.toLowerCase(), contains('moffett')); - expect(p.eventName, isNotEmpty); - } - }); - - // 5. Search driver wins - test('5. SEARCH_DRIVER_WINS: "Rallies Driver X won" returns only 1st place finishes on final stage', () async { - const query = SearchQuery( - intent: SearchIntent.searchDriverWins, - driverName: 'Josh Moffett', - ); - - final response = await repository.search(query); - expect(response.intent, equals(SearchIntent.searchDriverWins)); - expect(response.results, isA>()); - - final wins = response.results.cast(); - print('\nJosh Moffett won events: ${wins.length}'); - for (final w in wins) { - print(' • 🏆 ${w.eventName}: ${w.driverName} (Pos: ${w.posOverall}, Time: ${w.totalTime}s)'); - expect(w.posOverall, equals(1)); - } - }); - - // 6. Get first-place finisher / winner of a rally - test('6. GET_RALLY_RESULTS: "Winner of Rally X" returns first-place result on final classification', () async { - const query = SearchQuery( - intent: SearchIntent.getRallyResults, - rallyName: 'Moonraker Forestry Rally', - ); - - final response = await repository.search(query); - expect(response.intent, equals(SearchIntent.getRallyResults)); - expect(response.results, isA>()); - expect(response.results.length, equals(1)); - - final winner = response.results.cast().first; - print('\nWinner of Moonraker Rally: ${winner.positionBadge} ${winner.driverName} (Time: ${winner.totalTime}s)'); - expect(winner.posOverall, equals(1)); - expect(winner.driverName, isNotEmpty); - }); - - // 7. Get top 10 finishers of a rally (Leaderboard) - test('7. GET_RALLY_TOP_FINISHERS: "Top 10 finishers of Rally X" returns ranked classification table', () async { - const query = SearchQuery( - intent: SearchIntent.getRallyTopFinishers, - rallyName: 'Moonraker Forestry Rally', - limit: 10, - ); - - final response = await repository.search(query); - expect(response.intent, equals(SearchIntent.getRallyTopFinishers)); - expect(response.results, isA>()); - - final finishers = response.results.cast(); - expect(finishers.length, greaterThanOrEqualTo(3)); - print('\nMoonraker Rally Top Finishers:'); - for (int i = 0; i < finishers.length; i++) { - final f = finishers[i]; - print(' ${f.posOverall}. ${f.driverName} #${f.carNumber ?? ""} (${f.make ?? ""}) - ${f.totalTime}s'); - expect(f.posOverall, equals(i + 1)); - } - }); - - // 8. Search video action highlights for a rally - test('8. SEARCH_VIDEO_ACTIONS: "Jump highlights from Rally X" returns playable VideoAction moments', () async { - const query = SearchQuery( - intent: SearchIntent.searchVideoActions, - actionType: 'drift', - ); - - final response = await repository.search(query); - expect(response.intent, equals(SearchIntent.searchVideoActions)); - expect(response.results, isA>()); - expect(response.totalCount, greaterThan(0)); - - final actions = response.results.cast(); - final first = actions.first; - print('\nAction Moment: ${first.title} (${first.formattedDuration}), Stream URL: ${first.videoUrl}'); - expect(first.videoUrl, isNotNull); - expect(first.duration, greaterThan(0)); - }); - - // 9. Search videos featuring a driver - test('9. SEARCH_DRIVER_VIDEOS: "Videos featuring Driver X" uses explicit database links', () async { - const query = SearchQuery( - intent: SearchIntent.searchDriverVideos, - driverName: 'Philip Squires', - ); - - final response = await repository.search(query); - expect(response.intent, equals(SearchIntent.searchDriverVideos)); - expect(response.results, isA>()); - expect(response.totalCount, greaterThan(0)); - - final vids = response.results.cast(); - for (final v in vids) { - print(' • Video #${v.videoId} featuring ${v.driverName} in ${v.eventName} (${v.stageName})'); - expect(v.driverName?.toLowerCase(), contains('squires')); - expect(v.videoUrl, isNotNull); - } - }); - - // 10. Get top uploaders - test('10. GET_TOP_UPLOADERS: "Top uploaders" returns ranked contributors by upload count', () async { - const query = SearchQuery( - intent: SearchIntent.getTopUploaders, - limit: 5, - ); - - final response = await repository.search(query); - expect(response.intent, equals(SearchIntent.getTopUploaders)); - expect(response.results, isA>()); - expect(response.totalCount, greaterThan(0)); - - final uploaders = response.results.cast(); - print('\nTop Uploaders:'); - for (int i = 0; i < uploaders.length; i++) { - final u = uploaders[i]; - print(' #${i + 1} ${u.uploaderName}: ${u.uploadCount} uploads'); - expect(u.uploadCount, greaterThan(0)); - } - }); - - // 11. Get drivers with most career wins - test('11. GET_TOP_DRIVERS_BY_WINS: "Drivers with most wins" returns career winners leaderboard (1 win per event)', () async { - const query = SearchQuery( - intent: SearchIntent.getTopDriversByWins, - limit: 5, - ); - - final response = await repository.search(query); - expect(response.intent, equals(SearchIntent.getTopDriversByWins)); - expect(response.results, isA>()); - expect(response.totalCount, greaterThan(0)); - - final winners = response.results.cast(); - print('\nDrivers with Most Wins:'); - for (int i = 0; i < winners.length; i++) { - final w = winners[i]; - print(' #${i + 1} ${w.driverName}: ${w.winCount} wins (Latest: ${w.latestRallyWon})'); - expect(w.winCount, greaterThanOrEqualTo(1)); - } - }); - - // 12. Empty results handling - test('12. Empty results for non-matching criteria returns empty list without error', () async { - const query = SearchQuery( - intent: SearchIntent.searchRallies, - country: 'NonExistentCountryXYZ999', - ); - - final response = await repository.search(query); - expect(response.results, isEmpty); - expect(response.totalCount, equals(0)); - expect(response.hasMore, isFalse); - }); - - // 13. Pagination works deterministically - test('13. Pagination: limit & offset work deterministically', () async { - const page1Query = SearchQuery( - intent: SearchIntent.searchRallies, - limit: 3, - offset: 0, - ); - const page2Query = SearchQuery( - intent: SearchIntent.searchRallies, - limit: 3, - offset: 3, - ); - - final p1 = await repository.searchRallies(page1Query); - final p2 = await repository.searchRallies(page2Query); - - expect(p1.results.length, equals(3)); - expect(p2.results.length, equals(3)); - expect(p1.results.first.eventId, isNot(equals(p2.results.first.eventId))); - }); - }); - - group('Compound / Multi-Constraint Search Integration Tests', () { - // 1. Country + Year: "Show rallies in Ireland in 2025" - test('1. Compound: Country + Year (Ireland + 2025)', () async { - const query = SearchQuery( - intent: SearchIntent.searchRallies, - country: 'Ireland', - year: 2025, - ); - - final response = await repository.search(query); - expect(response.intent, equals(SearchIntent.searchRallies)); - expect(response.results, isA>()); - expect(response.results, isNotEmpty); - - final rallies = response.results.cast(); - print('\nRallies in Ireland in 2025: ${rallies.length}'); - for (final r in rallies) { - expect(r.country?.toLowerCase(), contains('ireland')); - expect( - r.year == 2025 || (r.startDate != null && r.startDate!.year == 2025) || (r.endDate != null && r.endDate!.year == 2025), - isTrue, - ); - } - }); - - // 2. Country + Driver: "Show rallies in Ireland where Josh Moffett participated" - test('2. Compound: Country + Driver (Ireland + Josh Moffett)', () async { - const query = SearchQuery( - intent: SearchIntent.searchDriverRallies, - country: 'Ireland', - driverName: 'Josh Moffett', - ); - - final response = await repository.search(query); - expect(response.intent, equals(SearchIntent.searchDriverRallies)); - expect(response.results, isA>()); - expect(response.results, isNotEmpty); - - final participations = response.results.cast(); - print('\nJosh Moffett participations in Ireland: ${participations.length}'); - for (final p in participations) { - expect(p.country?.toLowerCase(), contains('ireland')); - expect(p.driverName.toLowerCase(), contains('moffett')); - } - }); - - // 3. Country + Year + Driver: "Show rallies in Ireland in 2026 where Josh Moffett participated" - test('3. Compound: Country + Year + Driver (Ireland + 2026 + Josh Moffett)', () async { - const query = SearchQuery( - intent: SearchIntent.searchDriverRallies, - country: 'Ireland', - year: 2026, - driverName: 'Josh Moffett', - ); - - final response = await repository.search(query); - expect(response.intent, equals(SearchIntent.searchDriverRallies)); - expect(response.results, isA>()); - expect(response.results, isNotEmpty); - - final participations = response.results.cast(); - print('\nJosh Moffett participations in Ireland in 2026: ${participations.length}'); - for (final p in participations) { - expect(p.country?.toLowerCase(), contains('ireland')); - expect(p.driverName.toLowerCase(), contains('moffett')); - expect(p.startDate?.year, equals(2026)); - } - }); - - // 4. Event + Action: "Show drift highlights from Get Jerky Rally North Wales" - test('4. Compound: Event + Action (Get Jerky + drift)', () async { - const query = SearchQuery( - intent: SearchIntent.searchVideoActions, - eventName: 'Get Jerky', - actionType: 'drift', - ); - - final response = await repository.search(query); - expect(response.intent, equals(SearchIntent.searchVideoActions)); - expect(response.results, isA>()); - expect(response.results, isNotEmpty); - - final actions = response.results.cast(); - print('\nDrift highlights in Get Jerky: ${actions.length}'); - for (final a in actions) { - expect(a.actionType.toLowerCase(), contains('drift')); - expect(a.eventName?.toLowerCase(), contains('get jerky')); - } - }); - - // 5. Event + Action + Driver: "Show drift highlights featuring Philip Squires from Get Jerky Rally North Wales" - test('5. Compound: Event + Action + Driver (Get Jerky + drift + Philip Squires)', () async { - const query = SearchQuery( - intent: SearchIntent.searchVideoActions, - eventName: 'Get Jerky', - actionType: 'drift', - driverName: 'Philip Squires', - ); - - final response = await repository.search(query); - expect(response.intent, equals(SearchIntent.searchVideoActions)); - expect(response.results, isA>()); - - final actions = response.results.cast(); - print('\nPhilip Squires drift highlights in Get Jerky: ${actions.length}'); - for (final a in actions) { - expect(a.actionType.toLowerCase(), contains('drift')); - expect(a.eventName?.toLowerCase(), contains('get jerky')); - expect(a.driverName?.toLowerCase(), contains('squires')); - } - }); - - // 6. Event + Stage + Action: "Show drift highlights from Trackrod Rally 2024 Stage Gale Rigg" - test('6. Compound: Event + Stage + Action (Trackrod + Gale Rigg + drift)', () async { - const query = SearchQuery( - intent: SearchIntent.searchVideoActions, - eventName: 'Trackrod Rally', - stageName: 'Gale Rigg', - actionType: 'drift', - ); - - final response = await repository.search(query); - expect(response.intent, equals(SearchIntent.searchVideoActions)); - expect(response.results, isA>()); - expect(response.results, isNotEmpty); - - final actions = response.results.cast(); - print('\nTrackrod Gale Rigg drift highlights: ${actions.length}'); - for (final a in actions) { - expect(a.actionType.toLowerCase(), contains('drift')); - expect(a.eventName?.toLowerCase(), contains('trackrod')); - expect(a.stageName?.toLowerCase(), contains('gale rigg')); - } - }); - - - // 7. Driver + Year: "Show rallies won by Josh Moffett in 2026" - test('7. Compound: Driver + Year (Josh Moffett + 2026 Wins)', () async { - const query = SearchQuery( - intent: SearchIntent.searchDriverWins, - driverName: 'Josh Moffett', - year: 2026, - ); - - final response = await repository.search(query); - expect(response.intent, equals(SearchIntent.searchDriverWins)); - expect(response.results, isA>()); - expect(response.results, isNotEmpty); - - final wins = response.results.cast(); - print('\nJosh Moffett 2026 wins: ${wins.length}'); - for (final w in wins) { - expect(w.driverName.toLowerCase(), contains('moffett')); - expect(w.posOverall, equals(1)); - expect(w.startDate?.year, equals(2026)); - } - }); - - // 8. Empty compound results - test('8. Compound: Empty results for contradictory criteria returns gracefully', () async { - const query = SearchQuery( - intent: SearchIntent.searchDriverRallies, - country: 'Ireland', - year: 1950, - driverName: 'Josh Moffett', - ); - - final response = await repository.search(query); - expect(response.intent, equals(SearchIntent.searchDriverRallies)); - expect(response.results, isEmpty); - expect(response.totalCount, equals(0)); - expect(response.hasMore, isFalse); - }); - }); -} - diff --git a/test/services/llm/entity_resolution/generalized_stt_entity_resolution_test.dart b/test/services/llm/entity_resolution/generalized_stt_entity_resolution_test.dart deleted file mode 100644 index d9c98a4..0000000 --- a/test/services/llm/entity_resolution/generalized_stt_entity_resolution_test.dart +++ /dev/null @@ -1,229 +0,0 @@ -// ignore_for_file: avoid_print -import 'package:flutter_test/flutter_test.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:ai_rally_search/services/database_service.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/entity_lookup_repository.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/database_entity_resolver.dart'; -import 'package:ai_rally_search/services/llm/natural_language_search_service.dart'; -import 'package:ai_rally_search/services/llm/providers/mock_query_parser.dart'; -import 'package:ai_rally_search/models/entity_candidate.dart'; -import 'package:ai_rally_search/models/search_intent.dart'; -import 'package:ai_rally_search/models/search_query.dart'; - -void main() { - TestWidgetsFlutterBinding.ensureInitialized(); - - group('Generalized STT Entity Resolution & Recovery Test Suite', () { - late DatabaseService dbService; - late DatabaseEntityLookupRepository lookupRepo; - late DatabaseEntityResolver resolver; - - setUpAll(() async { - await dotenv.load(fileName: '.env'); - dbService = DatabaseService(); - lookupRepo = DatabaseEntityLookupRepository(dbService: dbService); - resolver = DatabaseEntityResolver( - repository: lookupRepo, - minConfidenceThreshold: 0.75, - minScoreGap: 0.15, - ); - }); - - tearDownAll(() async { - await dbService.close(); - }); - - group('1. Alūksne Voice STT Perturbations', () { - final aluksneVariants = [ - 'alux new', - 'eluksne', - 'aluknse', - 'aluxne', - ]; - - for (final variant in aluksneVariants) { - test('Variant "$variant" retrieves Rally Alūksne in Recall@5 and auto-resolves or suggests correctly', () async { - final candidates = await lookupRepo.lookupRallies(variant, limit: 5); - expect(candidates, isNotEmpty); - final hasCanonical = candidates.any((c) => c.canonicalName.contains('Alūksne')); - expect(hasCanonical, isTrue, reason: 'Rally Alūksne must appear in Recall@5 for "$variant"'); - - final query = SearchQuery( - intent: SearchIntent.searchVideoActions, - actionTypes: const ['action'], - rallyNames: [variant], - ); - - final result = await resolver.resolve(query); - if (result.requiresClarification) { - expect(result.candidates.any((c) => c.canonicalName.contains('Alūksne')), isTrue); - } else { - expect(result.resolvedQuery?.targetRallyName, contains('Alūksne')); - } - }); - } - }); - - group('2. Generalized Space-Boundary and Acoustic Entities', () { - test('Moonraker word-boundary join ("moon raker")', () async { - final candidates = await lookupRepo.lookupRallies('moon raker', limit: 5); - expect(candidates.any((c) => c.canonicalName.contains('Moonraker')), isTrue); - - final query = SearchQuery( - intent: SearchIntent.searchVideoActions, - actionTypes: const ['jump'], - rallyNames: const ['moon raker'], - ); - final result = await resolver.resolve(query); - final resolvedOrCandidate = result.requiresClarification - ? result.candidates.first.canonicalName - : (result.resolvedQuery?.targetRallyName ?? ''); - expect(resolvedOrCandidate, contains('Moonraker')); - }); - - test('Donegal word-boundary join ("done gal")', () async { - final candidates = await lookupRepo.lookupRallies('done gal', limit: 5); - expect(candidates.any((c) => c.canonicalName.contains('Donegal')), isTrue); - - final query = SearchQuery( - intent: SearchIntent.searchRallies, - rallyNames: const ['done gal'], - ); - final result = await resolver.resolve(query); - final resolvedOrCandidate = result.requiresClarification - ? result.candidates.first.canonicalName - : (result.resolvedQuery?.targetRallyName ?? ''); - expect(resolvedOrCandidate, contains('Donegal')); - }); - - test('Kemmelberg word-boundary and acoustic ("kemel berg")', () async { - final candidates = await lookupRepo.lookupStages('kemel berg', limit: 5); - expect(candidates.any((c) => c.canonicalName.contains('Kemmelberg')), isTrue); - - final query = SearchQuery( - intent: SearchIntent.searchVideoActions, - stageNames: const ['kemel berg'], - ); - final result = await resolver.resolve(query); - final resolvedOrCandidate = result.requiresClarification - ? result.candidates.first.canonicalName - : (result.resolvedQuery?.stageNames.first ?? ''); - expect(resolvedOrCandidate, contains('Kemmelberg')); - }); - - test('Duszniki acoustic distortion ("dushniki")', () async { - final candidates = await lookupRepo.lookupStages('dushniki', limit: 5); - expect(candidates.any((c) => c.canonicalName.contains('Duszniki')), isTrue); - - final query = SearchQuery( - intent: SearchIntent.searchVideoActions, - stageNames: const ['dushniki'], - ); - final result = await resolver.resolve(query); - final resolvedOrCandidate = result.requiresClarification - ? result.candidates.first.canonicalName - : (result.resolvedQuery?.stageNames.first ?? ''); - expect(resolvedOrCandidate, contains('Duszniki')); - }); - - test('Shea Breen vowel acoustic distortion ("Shea Brean")', () async { - final candidates = await lookupRepo.lookupDrivers('Shea Brean', limit: 15); - expect(candidates.any((c) => c.canonicalName == 'Shea Breen'), isTrue); - - final query = SearchQuery( - intent: SearchIntent.searchDriverVideos, - driverNames: const ['Shea Brean'], - ); - final result = await resolver.resolve(query); - final resolvedOrCandidate = result.requiresClarification - ? result.candidates.first.canonicalName - : (result.resolvedQuery?.driverName ?? ''); - expect(resolvedOrCandidate, equals('Shea Breen')); - }); - - test('Jon-Gunnar Støten diacritic/hyphen variant ("Jon Gunnar Stoten")', () async { - final candidates = await lookupRepo.lookupDrivers('Jon Gunnar Stoten', limit: 5); - expect(candidates.any((c) => c.canonicalName == 'Jon-Gunnar Støten'), isTrue); - - final query = SearchQuery( - intent: SearchIntent.searchDriverVideos, - driverNames: const ['Jon Gunnar Stoten'], - ); - final result = await resolver.resolve(query); - final resolvedOrCandidate = result.requiresClarification - ? result.candidates.first.canonicalName - : (result.resolvedQuery?.driverName ?? ''); - expect(resolvedOrCandidate, equals('Jon-Gunnar Støten')); - }); - }); - - group('3. Entity-Required Intents & Zero-Result Distinctions', () { - test('Condition A: Unidentifiable entity fails or clarifies without running raw un-resolved query', () async { - final query = SearchQuery( - intent: SearchIntent.searchVideoActions, - actionTypes: const ['jump'], - rallyNames: const ['zzqxxjjkww'], - ); - - final result = await resolver.resolve(query); - expect(result.isSuccess, isFalse); - if (result.requiresClarification) { - expect(result.candidates, isNotEmpty); - } else { - expect(result.error, contains('We couldn\'t confidently identify that rally')); - } - expect(result.resolvedQuery?.targetRallyName, isNull); - }); - - test('Condition B: Confidently resolved entity with 0 videos in DB produces clean search response', () async { - final query = SearchQuery( - intent: SearchIntent.searchVideoActions, - actionTypes: const ['crash'], - rallyNames: const ['Rally Alūksne 2026'], - ); - - final result = await resolver.resolve(query); - expect(result.isSuccess, isTrue); - expect(result.resolvedQuery?.targetRallyName, contains('Alūksne')); - }); - - test('Adversarial Negatives: Unrelated queries do NOT falsely auto-resolve', () async { - final negativeQueries = [ - 'Craig Nonexistentperson', - 'Rally Fakeplacenamexyz', - 'Completely Unrelated Noise', - ]; - - for (final neg in negativeQueries) { - final query = SearchQuery( - intent: SearchIntent.searchDriverVideos, - driverNames: [neg], - ); - final result = await resolver.resolve(query); - expect(result.resolvedQuery?.driverName == 'Craig Breen', isFalse); - expect(result.resolvedQuery?.driverName == 'Josh Moffett', isFalse); - if (result.isSuccess) { - expect(result.resolvedQuery?.driverName, isNot(contains('Breen'))); - } - } - }); - }); - - group('4. End-to-End Natural Language Flow', () { - test('Full query "rally alux new videos" resolves to Rally Alūksne', () async { - final mockParser = MockLlmQueryParser(); - final nlService = NaturalLanguageSearchService( - parser: mockParser, - entityResolver: resolver, - ); - - final nlResult = await nlService.search('rally alux new videos'); - if (nlResult.requiresClarification) { - expect(nlResult.candidates.any((c) => c.canonicalName.contains('Alūksne')), isTrue); - } else { - expect(nlResult.resolvedQuery?.targetRallyName, contains('Alūksne')); - } - }); - }); - }); -} diff --git a/test/services/llm/entity_resolution/live_database_entity_resolver_test.dart b/test/services/llm/entity_resolution/live_database_entity_resolver_test.dart deleted file mode 100644 index 2bb8ede..0000000 --- a/test/services/llm/entity_resolution/live_database_entity_resolver_test.dart +++ /dev/null @@ -1,126 +0,0 @@ -import 'package:flutter_test/flutter_test.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:ai_rally_search/models/entity_candidate.dart'; -import 'package:ai_rally_search/models/search_intent.dart'; -import 'package:ai_rally_search/models/search_query.dart'; -import 'package:ai_rally_search/services/database_service.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/database_entity_resolver.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/entity_lookup_repository.dart'; - -void main() { - TestWidgetsFlutterBinding.ensureInitialized(); - - group('Live AWS RDS MySQL Entity Resolution Integration Tests', () { - late DatabaseService db; - late DatabaseEntityLookupRepository lookupRepo; - late DatabaseEntityResolver resolver; - - setUpAll(() async { - await dotenv.load(fileName: '.env'); - db = DatabaseService(); - await db.connect(); - lookupRepo = DatabaseEntityLookupRepository(dbService: db); - resolver = DatabaseEntityResolver(repository: lookupRepo); - }); - - tearDownAll(() async { - await db.close(); - }); - - test('Live DB: "Moonraker" + year=2025 resolves Moonraker Forestry Rally 2025', () async { - const q = SearchQuery( - intent: SearchIntent.searchRallies, - rallyName: 'Moonraker', - year: 2025, - ); - - final res = await resolver.resolve(q); - expect(res.isSuccess, isTrue); - expect(res.requiresClarification, isFalse); - expect(res.resolvedQuery?.targetRallyName, contains('Moonraker Forestry Rally 2025')); - }); - - test('Live DB: "Moonraker" without year requires clarification for 2025 vs 2026 editions', () async { - const q = SearchQuery( - intent: SearchIntent.searchRallies, - rallyName: 'Moonraker', - ); - - final res = await resolver.resolve(q); - expect(res.requiresClarification, isTrue); - expect(res.candidates.length, greaterThanOrEqualTo(2)); - expect(res.candidates.any((c) => c.canonicalName.contains('2025')), isTrue); - expect(res.candidates.any((c) => c.canonicalName.contains('2026')), isTrue); - }); - - test('Live DB: "Get Jerky" resolves to Get Jerky Rally North Wales 2026', () async { - const q = SearchQuery( - intent: SearchIntent.searchRallies, - rallyName: 'Get Jerky', - ); - - final res = await resolver.resolve(q); - expect(res.isSuccess, isTrue); - expect(res.resolvedQuery?.targetRallyName, contains('Get Jerky Rally North Wales')); - }); - - test('Live DB: "Trackrod" resolves to Trackrod Rally 2024', () async { - const q = SearchQuery( - intent: SearchIntent.searchRallies, - rallyName: 'Trackrod', - ); - - final res = await resolver.resolve(q); - expect(res.isSuccess, isTrue); - expect(res.resolvedQuery?.targetRallyName, contains('Trackrod Rally')); - }); - - test('Live DB: "Josh Moffett" resolves to exact driver with UUID', () async { - const q = SearchQuery( - intent: SearchIntent.searchDriverRallies, - driverName: 'Josh Moffett', - ); - - final res = await resolver.resolve(q); - expect(res.isSuccess, isTrue); - expect(res.resolvedQuery?.driverName, 'Josh Moffett'); - expect(res.resolvedQuery?.driverId, isNotNull); - expect(res.resolvedQuery?.driverId!.isNotEmpty, isTrue); - }); - - test('Live DB: "Moffett" surname alone requires clarification across 6 drivers', () async { - const q = SearchQuery( - intent: SearchIntent.searchDriverRallies, - driverName: 'Moffett', - ); - - final res = await resolver.resolve(q); - expect(res.requiresClarification, isTrue); - expect(res.candidates.length, greaterThanOrEqualTo(4)); - }); - - test('Live DB: "Gale Rigg" stage resolves within Trackrod event context', () async { - const q = SearchQuery( - intent: SearchIntent.searchVideoActions, - rallyName: 'Trackrod', - stageName: 'Gale Rigg', - ); - - final res = await resolver.resolve(q); - expect(res.isSuccess, isTrue); - expect(res.resolvedQuery?.stageName, 'Gale Rigg'); - expect(res.resolvedQuery?.stageNumber, '3'); - }); - - test('Live DB: Unknown entity "Superman" does not invent IDs', () async { - const q = SearchQuery( - intent: SearchIntent.searchDriverRallies, - driverName: 'Superman', - ); - - final res = await resolver.resolve(q); - expect(res.resolvedQuery?.driverId, isNull); - expect(res.resolvedQuery?.driverName, 'Superman'); - }); - }); -} diff --git a/test/services/multi_value_database_search_test.dart b/test/services/multi_value_database_search_test.dart deleted file mode 100644 index 53108bd..0000000 --- a/test/services/multi_value_database_search_test.dart +++ /dev/null @@ -1,270 +0,0 @@ -import 'package:flutter_test/flutter_test.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:ai_rally_search/models/search_intent.dart'; -import 'package:ai_rally_search/models/search_query.dart'; -import 'package:ai_rally_search/services/database_service.dart'; -import 'package:ai_rally_search/services/search_repository.dart'; - -void main() { - TestWidgetsFlutterBinding.ensureInitialized(); - - late DatabaseService dbService; - late SearchRepository repository; - - setUpAll(() async { - await dotenv.load(fileName: '.env'); - dbService = DatabaseService(); - repository = SearchRepository(dbService: dbService); - }); - - tearDownAll(() async { - await dbService.close(); - }); - - group('Multi-Value Database Search & SQL Semantics Tests', () { - // 1. Multi-Action: jump OR drift - test('1. Multi-Action: actionTypes = ["jump", "drift"] returns union of actions', () async { - final query = SearchQuery( - intent: SearchIntent.searchVideoActions, - actionTypes: const ['jump', 'drift'], - limit: 20, - ); - - final response = await repository.searchVideoActions(query); - expect(response.results, isNotEmpty); - final actionTypesFound = response.results.map((a) => a.actionType.toLowerCase()).toSet(); - expect(actionTypesFound.every((a) => a.contains('jump') || a.contains('drift')), isTrue); - }); - - // 2. Multi-Country: Ireland OR United Kingdom - test('2. Multi-Country: countries = ["Ireland", "United Kingdom"] in rally search', () async { - final query = SearchQuery( - intent: SearchIntent.searchRallies, - countries: const ['Ireland', 'United Kingdom'], - limit: 20, - ); - - final response = await repository.searchRallies(query); - expect(response.results, isNotEmpty); - for (final r in response.results) { - final c = r.country?.toLowerCase() ?? ''; - expect( - c.contains('ireland') || c.contains('ie') || c.contains('united kingdom') || c.contains('gb') || c.contains('scotland') || c.contains('wales'), - isTrue, - ); - } - }); - - // 3. Multi-Year: 2024 OR 2025 - test('3. Multi-Year: years = [2024, 2025]', () async { - final query = SearchQuery( - intent: SearchIntent.searchRallies, - years: const [2024, 2025], - limit: 20, - ); - - final response = await repository.searchRallies(query); - expect(response.results, isNotEmpty); - for (final r in response.results) { - expect(r.year == 2024 || r.year == 2025, isTrue, reason: 'Failed for ${r.eventName} with year ${r.year}'); - } - }); - - // 4. Year Range: yearFrom = 2023, yearTo = 2025 - test('4. Year Range: yearFrom = 2023, yearTo = 2025', () async { - final query = SearchQuery( - intent: SearchIntent.searchRallies, - yearFrom: 2023, - yearTo: 2025, - limit: 20, - ); - - final response = await repository.searchRallies(query); - expect(response.results, isNotEmpty); - for (final r in response.results) { - expect(r.year, isNotNull); - expect(r.year!, inInclusiveRange(2023, 2025)); - } - }); - - // 5. Multi-Rally: Moonraker OR Trackrod - test('5. Multi-Rally: rallyNames = ["Moonraker", "Trackrod"]', () async { - final query = SearchQuery( - intent: SearchIntent.searchRallies, - rallyNames: const ['Moonraker', 'Trackrod'], - limit: 20, - ); - - final response = await repository.searchRallies(query); - expect(response.results, isNotEmpty); - for (final r in response.results) { - final name = r.eventName.toLowerCase(); - expect(name.contains('moonraker') || name.contains('trackrod'), isTrue); - } - }); - - // 6. Multi-Driver ANY: Josh Moffett OR Sam Moffett participations - test('6. Multi-Driver ANY: driverNames = ["Josh Moffett", "Sam Moffett"] with MatchMode.any', () async { - final query = SearchQuery( - intent: SearchIntent.searchDriverRallies, - driverNames: const ['Josh Moffett', 'Sam Moffett'], - driverMatchMode: MatchMode.any, - limit: 20, - ); - - final response = await repository.searchDriverRallies(query); - expect(response.results, isNotEmpty); - for (final r in response.results) { - final d = r.driverName?.toLowerCase() ?? ''; - expect(d.contains('josh') || d.contains('sam') || d.contains('moffett'), isTrue); - } - }); - - // 7. Multi-Driver ALL vs ANY: Prove live DB difference between ANY and ALL semantics - test('7. Multi-Driver ALL vs ANY: Explicitly proves MatchMode.any vs MatchMode.all difference on live DB', () async { - final queryAny = SearchQuery( - intent: SearchIntent.searchRallies, - driverNames: const ['Josh Moffett', 'Philip Squires'], - driverMatchMode: MatchMode.any, - limit: 50, - ); - - final queryAll = SearchQuery( - intent: SearchIntent.searchRallies, - driverNames: const ['Josh Moffett', 'Philip Squires'], - driverMatchMode: MatchMode.all, - limit: 50, - ); - - final responseAny = await repository.searchRallies(queryAny); - final responseAll = await repository.searchRallies(queryAll); - - // ANY returns the union of rallies where either Josh or Philip participated (e.g. Ireland + UK rallies) - expect(responseAny.results, isNotEmpty); - expect(responseAny.totalCount, greaterThan(0)); - - // ALL requires events where BOTH drivers competed simultaneously - expect(responseAny.totalCount, greaterThan(responseAll.totalCount)); - print('Live DB ANY vs ALL comparison: ANY count = ${responseAny.totalCount}, ALL count = ${responseAll.totalCount}'); - }); - - // 8. 3+ Dimensions Simultaneously: (jump OR drift) AND (Ireland OR United Kingdom) AND (2024 OR 2025) - test('8. 3+ Dimensions: actionTypes + countries + years combined', () async { - final query = SearchQuery( - intent: SearchIntent.searchVideoActions, - actionTypes: const ['jump', 'drift'], - countries: const ['Ireland', 'United Kingdom'], - years: const [2024, 2025], - limit: 20, - ); - - final response = await repository.searchVideoActions(query); - expect(response.results, isNotEmpty); - for (final a in response.results) { - expect(a.actionType.toLowerCase().contains('jump') || a.actionType.toLowerCase().contains('drift'), isTrue); - } - }); - - // 9. Pagination & Deduplication Stability: Distinct results across pages - test('9. Pagination & Deduplication: No duplicate records returned on multi-value joins', () async { - final queryP1 = SearchQuery( - intent: SearchIntent.searchVideoActions, - actionTypes: const ['jump', 'drift'], - countries: const ['Ireland', 'United Kingdom'], - limit: 5, - offset: 0, - ); - - final queryP2 = SearchQuery( - intent: SearchIntent.searchVideoActions, - actionTypes: const ['jump', 'drift'], - countries: const ['Ireland', 'United Kingdom'], - limit: 5, - offset: 5, - ); - - final p1 = await repository.searchVideoActions(queryP1); - final p2 = await repository.searchVideoActions(queryP2); - - final p1Ids = p1.results.map((a) => a.id).toSet(); - final p2Ids = p2.results.map((a) => a.id).toSet(); - - // Ensure no duplicates within page 1 or page 2 - expect(p1Ids.length, equals(p1.results.length)); - expect(p2Ids.length, equals(p2.results.length)); - // Ensure zero overlap between page 1 and page 2 - expect(p1Ids.intersection(p2Ids), isEmpty); - }); - - // 10. Zero results handled cleanly without error - test('10. Zero results for impossible multi-value filter returns empty list and count 0', () async { - final query = SearchQuery( - intent: SearchIntent.searchRallies, - countries: const ['NonexistentCountry123'], - years: const [1901], - ); - - final response = await repository.searchRallies(query); - expect(response.results, isEmpty); - expect(response.totalCount, equals(0)); - }); - - // 11. Pagination Determinism: 3-page pagination and count consistency - test('11. Pagination Determinism: Page 1, 2, 3 have disjoint canonical IDs and count matches total unique IDs', () async { - const pageSize = 5; - final qPage1 = SearchQuery( - intent: SearchIntent.searchVideoActions, - actionTypes: const ['jump', 'drift'], - limit: pageSize, - offset: 0, - ); - final qPage2 = SearchQuery( - intent: SearchIntent.searchVideoActions, - actionTypes: const ['jump', 'drift'], - limit: pageSize, - offset: pageSize, - ); - final qPage3 = SearchQuery( - intent: SearchIntent.searchVideoActions, - actionTypes: const ['jump', 'drift'], - limit: pageSize, - offset: pageSize * 2, - ); - final qTotal = SearchQuery( - intent: SearchIntent.searchVideoActions, - actionTypes: const ['jump', 'drift'], - limit: 100, - offset: 0, - ); - - final res1 = await repository.searchVideoActions(qPage1); - final res2 = await repository.searchVideoActions(qPage2); - final res3 = await repository.searchVideoActions(qPage3); - final resTotal = await repository.searchVideoActions(qTotal); - - final ids1 = res1.results.map((a) => a.id).toList(); - final ids2 = res2.results.map((a) => a.id).toList(); - final ids3 = res3.results.map((a) => a.id).toList(); - - // Ensure each page has distinct IDs - expect(ids1.toSet().length, equals(ids1.length)); - expect(ids2.toSet().length, equals(ids2.length)); - expect(ids3.toSet().length, equals(ids3.length)); - - // Ensure no overlap across pages - expect(ids1.toSet().intersection(ids2.toSet()), isEmpty); - expect(ids2.toSet().intersection(ids3.toSet()), isEmpty); - expect(ids1.toSet().intersection(ids3.toSet()), isEmpty); - - // Verify totalCount query equals the count returned by all queries - expect(res1.totalCount, equals(res2.totalCount)); - expect(res2.totalCount, equals(res3.totalCount)); - expect(res3.totalCount, equals(resTotal.totalCount)); - - // Verify ordering consistency with large limit - final combinedPaginatedIds = [...ids1, ...ids2, ...ids3]; - final totalSliceIds = resTotal.results.take(combinedPaginatedIds.length).map((a) => a.id).toList(); - expect(combinedPaginatedIds, equals(totalSliceIds)); - }); - }); -} diff --git a/test/services/person_participation_and_uploader_test.dart b/test/services/person_participation_and_uploader_test.dart deleted file mode 100644 index acadc09..0000000 --- a/test/services/person_participation_and_uploader_test.dart +++ /dev/null @@ -1,257 +0,0 @@ -// ignore_for_file: avoid_print -import 'package:flutter_test/flutter_test.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:ai_rally_search/services/database_service.dart'; -import 'package:ai_rally_search/services/search_repository.dart'; -import 'package:ai_rally_search/models/search_intent.dart'; -import 'package:ai_rally_search/models/search_query.dart'; -import 'package:ai_rally_search/models/search_results.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/entity_lookup_repository.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/database_entity_resolver.dart'; - -void main() { - TestWidgetsFlutterBinding.ensureInitialized(); - - setUpAll(() async { - await dotenv.load(fileName: '.env'); - }); - - group('Relational Source-of-Truth Regression Tests', () { - late DatabaseService db; - late SearchRepository repo; - late DatabaseEntityLookupRepository lookupRepo; - late DatabaseEntityResolver resolver; - - setUp(() { - db = DatabaseService(); - repo = SearchRepository(dbService: db); - lookupRepo = DatabaseEntityLookupRepository(dbService: db); - resolver = DatabaseEntityResolver(repository: lookupRepo); - }); - - tearDownAll(() async { - await db.close(); - }); - - test('1. Person existing only as Co-Driver (Max Freeman) - Entry List Participation', () async { - // Max Freeman participated in 7 distinct rally events according to rally_entry_list - final query = SearchQuery( - intent: SearchIntent.searchDriverRallies, - driverNames: ['Max Freeman'], - personRole: PersonRole.any, - ); - - final resp = await repo.searchDriverRallies(query); - print('\n[TEST 1] Max Freeman either-role participation: ${resp.results.length} events (totalCount: ${resp.totalCount})'); - for (final r in resp.results) { - print(' Event: ${r.eventName} | Role: ${r.role} | Car: ${r.car} #${r.carNumber} | Year: ${r.year}'); - } - - expect(resp.results.isNotEmpty, isTrue); - // In live DB, Max Freeman has entries in 7 distinct events: - expect(resp.totalCount, greaterThanOrEqualTo(7)); - expect(resp.results.any((r) => r.eventName.contains("Terras d'Aboboreira")), isTrue); - expect(resp.results.any((r) => r.eventName.contains("Circuit of Ireland")), isTrue); - expect(resp.results.any((r) => r.eventName.contains("Galway International")), isTrue); - expect(resp.results.any((r) => r.eventName.contains("Rally of the Lakes")), isTrue); - expect(resp.results.any((r) => r.eventName.contains("West Cork")), isTrue); - expect(resp.results.any((r) => r.eventName.contains("Down Rally")), isTrue); - expect(resp.results.any((r) => r.eventName.contains("Donegal")), isTrue); - - // Verify that role is Co-Driver - for (final r in resp.results) { - expect(r.role, equals('Co-Driver')); - } - }); - - test('2. Explicit Driver-only vs Co-Driver-only queries for Max Freeman', () async { - // Driver-only -> 0 events - final driverOnlyQuery = SearchQuery( - intent: SearchIntent.searchDriverRallies, - driverNames: ['Max Freeman'], - personRole: PersonRole.driver, - ); - final driverResp = await repo.searchDriverRallies(driverOnlyQuery); - expect(driverResp.results.isEmpty, isTrue); - expect(driverResp.totalCount, equals(0)); - - // Co-driver-only -> 7 events - final codriverOnlyQuery = SearchQuery( - intent: SearchIntent.searchDriverRallies, - driverNames: ['Max Freeman'], - personRole: PersonRole.coDriver, - ); - final codriverResp = await repo.searchDriverRallies(codriverOnlyQuery); - expect(codriverResp.results.isNotEmpty, isTrue); - expect(codriverResp.totalCount, greaterThanOrEqualTo(7)); - }); - - test('3. Person existing only as Driver (Josh Moffett)', () async { - final query = SearchQuery( - intent: SearchIntent.searchDriverRallies, - driverNames: ['Josh Moffett'], - personRole: PersonRole.any, - ); - final resp = await repo.searchDriverRallies(query); - print('\n[TEST 3] Josh Moffett participation: ${resp.results.length} events (totalCount: ${resp.totalCount})'); - expect(resp.results.isNotEmpty, isTrue); - expect(resp.totalCount, greaterThan(0)); - expect(resp.results.first.role, equals('Driver')); - }); - - test('4. searchRallies filters correctly for Co-Driver (Max Freeman)', () async { - final query = SearchQuery( - intent: SearchIntent.searchRallies, - driverNames: ['Max Freeman'], - personRole: PersonRole.any, - ); - final resp = await repo.searchRallies(query); - print('\n[TEST 4] searchRallies with Max Freeman: ${resp.results.length} events (totalCount: ${resp.totalCount})'); - expect(resp.results.isNotEmpty, isTrue); - expect(resp.totalCount, greaterThanOrEqualTo(7)); - expect(resp.results.any((r) => r.eventName.contains('Galway')), isTrue); - }); - - test('5. Sub-event deduplication: person in multiple sub-events of same event returns 1 event card', () async { - // In live DB, Donegal test rally & Wilton Donegal or similar sub-events map to events. - // Every result returned in searchDriverRallies must have a unique rallyId / event_id. - final query = SearchQuery( - intent: SearchIntent.searchDriverRallies, - driverNames: ['Max Freeman'], - personRole: PersonRole.any, - limit: 50, - ); - final resp = await repo.searchDriverRallies(query); - final eventIds = resp.results.map((r) => r.rallyId).toList(); - final uniqueEventIds = eventIds.toSet().toList(); - expect(eventIds.length, equals(uniqueEventIds.length), reason: 'Each rally event must appear at most once'); - }); - - test('6. Top Uploaders resolution: Canonical fan_id / account_id join with human names and profile picture', () async { - final query = SearchQuery( - intent: SearchIntent.getTopUploaders, - limit: 10, - ); - final resp = await repo.getTopUploaders(query); - print('\n[TEST 6] Top Uploaders: ${resp.results.length} items (totalCount: ${resp.totalCount})'); - for (final u in resp.results) { - print(' Uploader: "${u.uploaderName}" | Count: ${u.uploadCount} | Pic: ${u.profilePicture} | ID: ${u.uploaderId}'); - } - - expect(resp.results.isNotEmpty, isTrue); - expect(resp.totalCount, greaterThan(0)); - - // Check that names are NOT falling back to "Anonymous" when user profile/account exists - final namedUploaders = resp.results.where((u) => u.uploaderName != 'Anonymous' && u.uploaderName != 'Rally Contributor').toList(); - expect(namedUploaders.isNotEmpty, isTrue, reason: 'Should resolve real fan full_name or user_name'); - - // Verify canonical IDs - final uploaderIds = resp.results.map((u) => u.uploaderId).toList(); - expect(uploaderIds.toSet().length, equals(uploaderIds.length), reason: 'Each uploader must have a unique canonical identity'); - }); - - test('7. lookupUploaders correctly uses user_fan_profile and user_account', () async { - final matches = await lookupRepo.lookupUploaders('max'); - print('\n[TEST 7] lookupUploaders("max"): ${matches.length} candidates'); - for (final m in matches) { - print(' $m | sub: ${m.subtitle} | meta: ${m.metadata}'); - } - expect(matches.isNotEmpty, isTrue); - expect(matches.any((m) => m.canonicalName.toLowerCase().contains('max') || (m.subtitle?.toLowerCase().contains('max') ?? false)), isTrue); - }); - - test('8. Entity Resolution for Person in both roles attaches role: both', () async { - // Joshua Carr appears in live DB as both driver and codriver - final matches = await lookupRepo.lookupDrivers('Joshua Carr'); - print('\n[TEST 8] lookupDrivers("Joshua Carr"): ${matches.length} candidates'); - for (final m in matches) { - print(' $m | sub: ${m.subtitle} | meta: ${m.metadata}'); - } - expect(matches.isNotEmpty, isTrue); - final carr = matches.firstWhere((m) => m.canonicalName == 'Joshua Carr'); - expect(carr.metadata?['role'], equals('both')); - expect(carr.subtitle, contains('DRIVER / CO-DRIVER')); - }); - - test('9. Full Entity Resolution flow for Max Freeman', () async { - final inputQuery = SearchQuery( - intent: SearchIntent.searchDriverRallies, - driverNames: ['Max Freeman'], - ); - final resResult = await resolver.resolve(inputQuery); - print('\n[TEST 9] Resolved query for Max Freeman: ${resResult.resolvedQuery?.driverNames} | IDs: ${resResult.resolvedQuery?.driverIds}'); - expect(resResult.requiresClarification, isFalse); - expect(resResult.resolvedQuery, isNotNull); - expect(resResult.resolvedQuery!.driverNames, contains('Max Freeman')); - expect(resResult.resolvedQuery!.driverIds.isNotEmpty, isTrue); - expect(resResult.resolvedQuery!.driverIds.first, equals('7a633b52-950e-49ef-8cab-34cd43e99366')); - }); - - test('10. Uploader Name Fallback Order: user_name > full_name > email > Rally Contributor', () { - // 1. Both user_name and full_name present -> user_name wins - final dto1 = UploaderSearchResult.fromMap({ - 'uploader_user_id': 'fan-1', - 'user_name': 'jerry_rally_fan', - 'full_name': 'Jerry Lynch', - 'email': 'jerry@example.com', - 'upload_count': 10, - }); - expect(dto1.uploaderName, equals('jerry_rally_fan')); - - // 2. user_name is null/empty -> full_name wins - final dto2 = UploaderSearchResult.fromMap({ - 'uploader_user_id': 'fan-2', - 'user_name': ' ', - 'full_name': 'Jerry Lynch', - 'email': 'jerry@example.com', - 'upload_count': 10, - }); - expect(dto2.uploaderName, equals('Jerry Lynch')); - - // 3. Both user_name and full_name absent -> email wins - final dto3 = UploaderSearchResult.fromMap({ - 'uploader_user_id': 'fan-3', - 'user_name': '', - 'full_name': null, - 'email': 'jerry@example.com', - 'upload_count': 10, - }); - expect(dto3.uploaderName, equals('jerry@example.com')); - - // 4. All absent -> 'Rally Contributor' fallback - final dto4 = UploaderSearchResult.fromMap({ - 'uploader_user_id': 'fan-4', - 'user_name': '', - 'full_name': null, - 'email': ' ', - 'upload_count': 10, - }); - expect(dto4.uploaderName, equals('Rally Contributor')); - }); - - test('11. Different fan IDs are not merged even if they share username or display name', () { - final u1 = UploaderSearchResult.fromMap({ - 'uploader_user_id': 'fan-uuid-1', - 'user_name': 'max_rally', - 'upload_count': 10, - }); - final u2 = UploaderSearchResult.fromMap({ - 'uploader_user_id': 'fan-uuid-2', - 'user_name': 'max_rally', - 'upload_count': 5, - }); - expect(u1.uploaderId, isNot(equals(u2.uploaderId))); - expect(u1.uploaderName, equals(u2.uploaderName)); - }); - - test('12. Profile picture is preserved from user_fan_profile', () { - final u = UploaderSearchResult.fromMap({ - 'uploader_user_id': 'fan-pic-1', - 'user_name': 'Mad4TarRallying', - 'profile_picture': 'https://assets.prod.pineamite.com/profiles/fan/pic.jpg', - 'upload_count': 50, - }); - expect(u.profilePicture, equals('https://assets.prod.pineamite.com/profiles/fan/pic.jpg')); - }); - }); -} diff --git a/test/services/rally_streams_filter_test.dart b/test/services/rally_streams_filter_test.dart deleted file mode 100644 index 8d4752c..0000000 --- a/test/services/rally_streams_filter_test.dart +++ /dev/null @@ -1,20 +0,0 @@ -import 'package:flutter_test/flutter_test.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:ai_rally_search/services/database_service.dart'; - -void main() { - test('getRallyStreams excludes instantReplay video type rows', () async { - await dotenv.load(fileName: '.env'); - final db = DatabaseService(); - - final streams = await db.getRallyStreams(limit: 50); - expect(streams, isNotEmpty); - - for (final stream in streams) { - final videoType = stream['video_type']?.toString(); - expect(videoType, isNot(equals('instantReplay'))); - } - - await db.close(); - }); -} diff --git a/test/services/video_action_repository_test.dart b/test/services/video_action_repository_test.dart deleted file mode 100644 index af1496c..0000000 --- a/test/services/video_action_repository_test.dart +++ /dev/null @@ -1,27 +0,0 @@ -import 'package:flutter_test/flutter_test.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; -import 'package:ai_rally_search/services/database_service.dart'; -import 'package:ai_rally_search/services/video_action_repository.dart'; - -void main() { - test('VideoActionRepository fetches and parses live actions from database', () async { - await dotenv.load(fileName: '.env'); - final repo = VideoActionRepository(); - - final recentActions = await repo.getRecentVideoActions(limit: 5); - expect(recentActions, isA()); - print('Recent Actions Count: ${recentActions.length}'); - - if (recentActions.isNotEmpty) { - final first = recentActions.first; - print('First Action: ${first.title} (${first.actionType}), Range: ${first.formattedTimeRange}, Duration: ${first.formattedDuration}, Video #${first.videoId}, URL: ${first.videoUrl}'); - expect(first.id, greaterThan(0)); - expect(first.videoId, greaterThan(0)); - expect(first.actionType.isNotEmpty, true); - expect(first.startTime, greaterThanOrEqualTo(0)); - expect(first.endTime, greaterThanOrEqualTo(first.startTime)); - } - - await DatabaseService().close(); - }); -} diff --git a/test/widget_test.dart b/test/widget_test.dart index 722315a..1c67d10 100644 --- a/test/widget_test.dart +++ b/test/widget_test.dart @@ -10,7 +10,7 @@ void main() { testWidgets('App launches on the search-first home', ( WidgetTester tester, ) async { - await dotenv.load(fileName: '.env'); + await dotenv.load(fileName: 'assets/config/app_config.env'); await tester.pumpWidget(const MyApp()); await tester.pump(); diff --git a/tool/export_benchmarks.dart b/tool/export_benchmarks.dart deleted file mode 100644 index 331cbac..0000000 --- a/tool/export_benchmarks.dart +++ /dev/null @@ -1,114 +0,0 @@ -import 'dart:convert'; -import 'dart:io'; -import 'dart:math'; - -import 'package:ai_rally_search/services/database_service.dart'; -import 'package:ai_rally_search/services/entity_search/entity_search_models.dart'; -import 'package:ai_rally_search/services/entity_search/mysql_entity_search_data_source.dart'; -import 'package:ai_rally_search/services/llm/entity_resolution/phonetic_matching_helper.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; - -import '../test/eval/entity_search/deterministic_corruption_generator.dart'; -import '../test/eval/entity_search/held_out_entity_fixture.dart'; - -void main() async { - await dotenv.load(fileName: '.env'); - final database = DatabaseService(); - final source = MySqlEntitySearchDataSource(database: database); - final all = await source.loadEntities(); - await database.disconnect(); - - // 1. Export 803 cases - final selected80 = []; - final fixtureSeedIdByCanonicalId = {}; - for (final entry in heldOutEntityIds.entries) { - for (final id in entry.value) { - final matches = all - .where( - (e) => - e.entityType == entry.key && - (e.canonicalId == id || - (entry.key == SearchEntityType.person && - e.metadata['accountId']?.toString() == id)), - ) - .toList(); - if (matches.isNotEmpty) { - selected80.add(matches.single); - fixtureSeedIdByCanonicalId[matches.single.canonicalId] = id; - } - } - } - - final generator803 = DeterministicCorruptionGenerator(heldOutSeed); - final cases803 = >[]; - for (final target in selected80) { - final corruptions = generator803.generate( - target.canonicalName, - fixtureSeedIdByCanonicalId[target.canonicalId] ?? target.canonicalId, - person: target.entityType == SearchEntityType.person, - ); - for (final c in corruptions) { - cases803.add({ - 'targetCanonicalId': target.canonicalId, - 'targetCanonicalName': target.canonicalName, - 'entityType': target.entityType.name, - 'corruptionKind': c.kind, - 'difficulty': c.difficulty.name, - 'input': c.value, - }); - } - } - File('test/eval/entity_search/frozen_803_cases.json').writeAsStringSync( - const JsonEncoder.withIndent(' ').convert(cases803), - ); - print('Exported ${cases803.length} frozen 803 cases'); - - // 2. Export 1108 person cases - final people = all.where((e) => e.entityType == SearchEntityType.person).toList(); - final groups = >{ - 'ACCOUNT_BACKED': people.where((e) => e.metadata['identityKind'] == 'account').toList(), - 'NULL_DRIVER': people.where((e) => e.metadata['identityKind'] == 'driver').toList(), - 'NULL_CODRIVER': people.where((e) => e.metadata['identityKind'] == 'codriver').toList(), - }; - final desired = {'ACCOUNT_BACKED': 20, 'NULL_DRIVER': 40, 'NULL_CODRIVER': 40}; - final selectedPerson = >{}; - const excludedNames = {'pawel molgo', 'shea breen', 'max freeman', 'chris melly', 'melly'}; - const personSeed = 20260828; - - for (final entry in groups.entries) { - final eligible = entry.value.where((entity) { - final normalized = PhoneticMatchingHelper.normalize(entity.canonicalName); - return !excludedNames.any((ex) => normalized == ex || normalized.contains(ex)); - }).toList()..shuffle( - Random(personSeed + switch (entry.key) { 'ACCOUNT_BACKED' => 1, 'NULL_DRIVER' => 2, _ => 3 }), - ); - selectedPerson[entry.key] = eligible.take(desired[entry.key]!).toList(); - } - - final generatorPerson = DeterministicCorruptionGenerator(personSeed); - final cases1108 = >[]; - for (final entry in selectedPerson.entries) { - for (final target in entry.value) { - final corruptions = generatorPerson.generate( - target.canonicalName, - target.canonicalId, - person: true, - ); - for (final c in corruptions) { - cases1108.add({ - 'group': entry.key, - 'targetCanonicalId': target.canonicalId, - 'targetCanonicalName': target.canonicalName, - 'entityType': 'person', - 'corruptionKind': c.kind, - 'difficulty': c.difficulty.name, - 'input': c.value, - }); - } - } - } - File('test/eval/entity_search/frozen_1108_cases.json').writeAsStringSync( - const JsonEncoder.withIndent(' ').convert(cases1108), - ); - print('Exported ${cases1108.length} frozen 1108 cases'); -}