⚡ PERF: instrument search latency and add a 4s progressive offline fallback

.env.example

@@ -1,3 +1,19 @@
+# =============================================================================
+# SERVER-SIDE environment template (FastAPI backend + local dev tooling).
+# =============================================================================
+# Loaded ONLY by the backend (see backend/app/config.py) and local dev scripts.
+# Copy to `.env` and fill in real values. `.env` is gitignored and MUST NEVER
+# be committed or bundled into the Flutter app.
+#
+# The Flutter client does NOT read this file. Its only bundled configuration is
+# assets/config/app_config.env, which contains PUBLIC values only (API base
+# URL, non-secret speech settings) — never DB credentials or API keys. The
+# device reaches the database exclusively through the FastAPI backend over
+# HTTPS; provider API keys are used server-side (e.g. the voice-transcribe
+# proxy), never shipped to the device.
+# =============================================================================
+
+# --- Database (server-side ONLY; never bundled in the app) -------------------
 DB_HOST=
 DB_PORT=
 DB_NAME=
@@ -6,6 +22,20 @@ DB_PASSWORD=
 DB_USE_SSL=
 ENTITY_SEARCH_FALLBACK_MODE=
 
+# --- Observability -----------------------------------------------------------
+# Structured (JSON) request timing lines are always emitted; these tune them.
+LOG_FORMAT=json
+LOG_LEVEL=INFO
+# Attaches the per-phase timing breakdown to the search response body. Leave
+# off in production and correlate via the request id in the logs instead.
+EXPOSE_DEBUG_TIMINGS=false
+
+# --- Database connection pool ------------------------------------------------
+DB_POOL_SIZE=5
+DB_MAX_OVERFLOW=5
+DB_POOL_TIMEOUT_SECONDS=10
+DB_POOL_RECYCLE_SECONDS=1800
+
 QUERY_UNDERSTANDING_PROVIDER=gemini
 QUERY_UNDERSTANDING_MODEL=gemini-3.5-flash-lite
 GEMINI_API_KEY=your_api_key_here

assets/config/app_config.env

@@ -0,0 +1,26 @@
+# =============================================================================
+# PUBLIC CLIENT CONFIGURATION — ships inside the app bundle.
+# =============================================================================
+# This file is compiled into the Flutter app and is readable by anyone who
+# installs it. It MUST contain ONLY non-sensitive, public configuration.
+#
+# NEVER put secrets here (or in any client-bundled file):
+#   - No DB_HOST / DB_PORT / DB_USER / DB_PASSWORD / DATABASE_URL
+#   - No OPENAI_API_KEY / GEMINI_API_KEY / ANTHROPIC_API_KEY
+#   - No AWS keys, tokens, or passwords
+#
+# The device never connects to the database directly. All data access goes
+# through the FastAPI backend over HTTPS. Database credentials and provider
+# API keys live ONLY in server-side environment variables (see backend/.env
+# / .env.example, loaded by backend/app/config.py).
+# =============================================================================
+
+# Base URL of the FastAPI backend. This is the single piece of config the
+# client needs. Override per build/flavor as appropriate (public value).
+PYTHON_BACKEND_BASE_URL=http://0.0.0.0:8765/
+
+# Speech provider selection (non-secret). Cloud transcription is proxied
+# through the backend's /v1/voice/transcribe endpoint, so no API key is
+# required on the device.
+SPEECH_PROVIDER=openai_proxy
+SPEECH_MODEL=whisper-1

backend/app/api/v1/conversation.py

@@ -1,8 +1,11 @@
+from functools import lru_cache
+
 from fastapi import APIRouter, Depends
 from pydantic import BaseModel, ConfigDict, Field
 from sqlalchemy.ext.asyncio import AsyncConnection
 
 from ...config import Settings, get_settings
+from ...observability import Phase, current_timings
 from ...db.engine import get_connection
 from ...domain.conversation_session import SearchConversationSession
 from ...entity_search.adapter import EntitySearchLookupAdapter
@@ -13,7 +16,11 @@
 from ...query_understanding.providers import AnthropicProvider, GeminiProvider, MockProvider, OpenAIProvider
 from ...query_understanding.service import QueryUnderstandingService
 from ...repositories.search_repository import SearchRepository
-from ...services.conversational_search_service import ConversationalSearchResult, ConversationalSearchService
+from ...services.conversational_search_service import (
+    ConversationalSearchResult,
+    ConversationalSearchService,
+    record_result_attributes,
+)
 
 router = APIRouter(prefix="/v1/conversation")
 
@@ -33,6 +40,30 @@ class ConversationSearchResponse(BaseModel):
     session: SearchConversationSession
     result: ConversationalSearchResult
     request_id: int | None = Field(default=None, alias="requestId")
+    trace_id: str | None = Field(default=None, alias="traceId")
+
+
+@lru_cache(maxsize=4)
+def _cached_query_service(
+    provider: str,
+    model: str,
+    api_key: str | None,
+    base_url: str | None,
+    temperature: float,
+    timeout_seconds: float,
+    max_retries: int,
+    provider_type: type,
+) -> QueryUnderstandingService:
+    config = ProviderConfig(
+        provider=provider,
+        model=model,
+        api_key=api_key,
+        base_url=base_url,
+        temperature=temperature,
+        timeout_seconds=timeout_seconds,
+        max_retries=max_retries,
+    )
+    return QueryUnderstandingService(provider_type(config))
 
 
 def _build_query_service(settings: Settings) -> QueryUnderstandingService:
@@ -83,16 +114,19 @@ def _build_query_service(settings: Settings) -> QueryUnderstandingService:
                 "missing. Provide the API key or change the provider."
             )
 
-    config = ProviderConfig(
-        provider=name,
-        model=settings.query_understanding_model,
-        api_key=api_key,
-        base_url=urls.get(name),
-        temperature=settings.query_understanding_temperature,
-        timeout_seconds=settings.query_understanding_timeout_seconds,
-        max_retries=settings.query_understanding_max_retries,
+    # Validation above runs on every call and still fails fast; only the
+    # construction of the (stateless) service and provider is memoised, so
+    # each request no longer rebuilds them.
+    return _cached_query_service(
+        name,
+        settings.query_understanding_model,
+        api_key,
+        urls.get(name),
+        settings.query_understanding_temperature,
+        settings.query_understanding_timeout_seconds,
+        settings.query_understanding_max_retries,
+        provider_types[name],
     )
-    return QueryUnderstandingService(provider_types[name](config))
 
 
 from ...entity_search.warmup import (
@@ -122,14 +156,28 @@ async def get_conversational_service(
 async def search_conversation(
     request: ConversationSearchRequest,
     service: ConversationalSearchService = Depends(get_conversational_service),
+    settings: Settings = Depends(get_settings),
 ) -> ConversationSearchResponse:
+    timings = current_timings()
+    if timings is not None:
+        # Everything before the handler body — dependency resolution, which is
+        # where the DB connection is checked out and the entity index awaited.
+        timings.add(
+            Phase.DEPENDENCIES,
+            max(0.0, timings.total_ms - sum(timings.phases.values())),
+        )
     updated_session, result = await service.search(
         request.query,
         session=request.session,
         language=request.language,
+        expose_timings=settings.expose_debug_timings,
     )
+    if timings is not None:
+        record_result_attributes(timings, result)
+        timings.mark_handler_complete()
     return ConversationSearchResponse(
         session=updated_session,
         result=result,
         request_id=request.request_id,
+        trace_id=result.request_id,
     )

backend/app/config.py

@@ -43,6 +43,16 @@ def _apply_env_fallbacks(cls, data: Any) -> Any:
     db_user: str = ""
     db_password: SecretStr = SecretStr("")
     db_use_ssl: bool = False
+    # Connection pooling. Sized for a small Railway container: enough warm
+    # connections to absorb bursts without exhausting MySQL's max_connections
+    # across workers. `recycle` stays well under a typical 8h wait_timeout.
+    db_pool_size: int = 5
+    db_max_overflow: int = 5
+    db_pool_timeout_seconds: float = 10.0
+    db_pool_recycle_seconds: int = 1800
+    # Per-request phase timings are attached to the response only when this is
+    # on. Production correlates via request_id in structured logs instead.
+    expose_debug_timings: bool = False
     entity_search_fallback_mode: str = "FALLBACK"
     query_understanding_provider: str = "mock"
     query_understanding_model: str = "mock-parser-v1"

backend/app/db/engine.py

@@ -1,20 +1,62 @@
 from collections.abc import AsyncIterator
+
 from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, create_async_engine
-from sqlalchemy.pool import NullPool
+
 from ..config import get_settings
+from ..observability import Phase, current_timings
 
 _engine: AsyncEngine | None = None
 
+
 def get_engine() -> AsyncEngine:
     global _engine
     if _engine is None:
         settings = get_settings()
         if not settings.db_host or not settings.db_name or not settings.db_user:
             raise RuntimeError("Database configuration is incomplete")
-        # Async connections must never leak across worker/event-loop boundaries.
-        _engine = create_async_engine(settings.database_url, poolclass=NullPool)
+        # A pooled engine. The previous NullPool opened a fresh TCP + TLS +
+        # MySQL auth handshake on every request, measured at ~178 ms each
+        # against the production host; a checked-out pooled connection costs
+        # ~30 ms. `pool_pre_ping` covers connections the server closed while
+        # idle, and `pool_recycle` stays below the usual wait_timeout so a
+        # stale connection is never handed to a request.
+        #
+        # Async connections must still never leak across event loops: the
+        # engine is created lazily per process, and every worker process
+        # therefore builds its own pool.
+        _engine = create_async_engine(
+            settings.database_url,
+            pool_size=settings.db_pool_size,
+            max_overflow=settings.db_max_overflow,
+            pool_timeout=settings.db_pool_timeout_seconds,
+            pool_recycle=settings.db_pool_recycle_seconds,
+            pool_pre_ping=True,
+        )
     return _engine
 
+
+async def dispose_engine() -> None:
+    global _engine
+    if _engine is not None:
+        await _engine.dispose()
+        _engine = None
+
+
 async def get_connection() -> AsyncIterator[AsyncConnection]:
-    async with get_engine().connect() as connection:
+    """Yields a pooled connection, recording checkout cost as `db_connect`.
+
+    Dependency resolution runs before the handler body, so without this
+    measurement the connection cost lands outside every in-handler timer.
+    """
+    timings = current_timings()
+    if timings is None:
+        async with get_engine().connect() as connection:
+            yield connection
+        return
+    with timings.measure(Phase.DB_CONNECT):
+        context = get_engine().connect()
+        connection = await context.__aenter__()
+    try:
         yield connection
+    finally:
+        await context.__aexit__(None, None, None)

backend/app/main.py

@@ -10,26 +10,36 @@
 from .api.v1.query_understanding import router as query_understanding_router
 from .api.v1.search import router as search_router
 from .api.v1.voice import router as voice_router
-from .db.engine import get_engine
+from .db.engine import dispose_engine, get_engine
 from .domain.errors import ApiError, ErrorCode
 from .entity_search.warmup import (
     cancel_background_entity_search_warmup,
     start_background_entity_search_warmup,
 )
+from .observability.logging import configure_logging
+from .observability.middleware import timing_middleware
+from .query_understanding.providers.http import close_client
 
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
+    # Lift app loggers above uvicorn's default config. Without this the
+    # OpenEntity warm-up and per-request timing lines never reach stdout, which
+    # is why cold-start cost was previously invisible in production.
+    configure_logging()
     # Non-blocking OpenEntity background warm-up
     engine = get_engine()
     start_background_entity_search_warmup(engine)
     try:
         yield
     finally:
         await cancel_background_entity_search_warmup()
+        await close_client()
+        await dispose_engine()
 
 
 app = FastAPI(title="AI Rally Search deterministic backend", version="0.1.0", lifespan=lifespan)
+app.middleware("http")(timing_middleware)
 app.include_router(health_router)
 app.include_router(search_router)
 app.include_router(offline_router)

backend/app/observability/__init__.py

@@ -0,0 +1,23 @@
+"""Latency observability: request identity, phase timings, structured logs.
+
+Nothing in here records raw user text, credentials, or result payloads. The
+only free-form value that ever reaches a log line is the request id, which the
+client generates and which carries no user data.
+"""
+
+from .request_context import (
+    current_request_id,
+    current_timings,
+    new_request_id,
+    request_scope,
+)
+from .timings import Phase, RequestTimings
+
+__all__ = [
+    "Phase",
+    "RequestTimings",
+    "current_request_id",
+    "current_timings",
+    "new_request_id",
+    "request_scope",
+]