config-example.yml

# Locale for CLI/UI messages. Falls back to CODEXA_LOCALE / LANG / "en" when
# unset. Catalogs live in codexa/locale/<code>/LC_MESSAGES/codexa.po.
# Internal log event keys (run_complete, plugin_loaded, …) stay English
# regardless — only the human-readable strings are translated.
locale: "en"

# Local operator-data retention / minimisation. `0` means "keep
# indefinitely" except `log_retention_days`, whose default 7 mirrors the
# rotating indexer log handler. `redact_log_paths` replaces private
# absolute paths in structured logs and export path fields with basenames.
# `redact_export_content` omits query, answer, and passage body fields
# from JSON exports while preserving metadata/provenance fields.
privacy:
  log_retention_days: 7
  redact_log_paths: false
  redact_export_content: false
  pending_wal_retention_days: 30
  pending_wal_max_bytes: 1048576
  chat_retention_days: 0
  pilot_retention_days: 0
  query_history_enabled: true
  query_history_retention_days: 0

data_dirs:
  - "/path/to/corpus"

# Ingestion-time routing knobs.
ingestion:
  # CL-16: per-chunk language stamp confidence gate. 0.0 stamps every
  # chunk; raise toward 1.0 to keep low-signal chunks on the file-level
  # dominant-language fallback while recording language_low_confidence.
  lang_detect:
    min_confidence: 0.0

# Vector store backends are built-in adapters.
# Plugins can register future drop-ins (FAISS, Qdrant, LanceDB, …) via
# `codexa.store.register_store_backend(name, factory)`, after which
# validate-config accepts that backend name. The `persist_dir` /
# `collection_name` / `file_collection` / `entity_collection` keys are
# Chroma-specific; alternative backends may read different keys.
store:
  # "sharded_chroma" (safe new-install default) or "chroma" (legacy
  # single sqlite/HNSW). The sharded backend spreads the corpus across
  # per-file-type / first-character sqlite
  # files like `persist_dir/pdf_a_0000000000.sqlite3`. Chroma still
  # gets a per-shard segment dir, with its hardcoded `chroma.sqlite3`
  # path symlinked to that named file. Once a bucket sqlite reaches
  # 1 GiB, the next write for that bucket rolls to `...0000000001`,
  # then `...0000000002`, etc. Special first characters route to `_`,
  # e.g. `pdf___0000000000.sqlite3`.
  backend: "sharded_chroma"
  persist_dir: "./chroma_db"
  # sharded_chroma reads scatter-gather across every shard. Migrate by
  # re-indexing into a fresh sharded persist_dir.
  # Each Chroma shard owns native threads and file descriptors. Clients
  # are opened on demand and idle clients are closed above this LRU cap.
  max_open_shards: 16
  # Tokio workers per embedded Chroma client and the process-wide Rayon
  # cap. Keep this low: max_open_shards multiplies the Tokio runtime cost.
  chroma_runtime_threads: 1
  # Chroma-backed collection names: 3-512 characters from
  # [a-zA-Z0-9._-], starting and ending with a letter or digit.
  # validate-config enforces this so a bad name fails before a run
  # rather than per file at embed time.
  collection_name: "documents"
  entity_collection: "entities"
  file_collection: "files"
  # SD-12 (ADR 0001) ingest-vs-serve split. Requires
  # `backend: "sharded_chroma"`. `publish: "generations"` stages
  # each index run in `gen_NNNNNN/` and publish atomically via a CURRENT
  # pointer so readers never observe a half-written store. Existing configs
  # that explicitly use `backend: "chroma"` remain in-place. The retention
  # window keeps the latest N published generations for rollback.
  publish: "generations"
  # Snapshot-backed interactive queries require an eligible published CURRENT.
  # `current` is the published-serving default: a re-index builds in staging
  # and swaps atomically, so one query never sees half-written state.
  # `serve: live` is rejected by those immutable query paths because mutable
  # staging cannot satisfy a query snapshot.
  # On a first build, wait for the first published checkpoint before searching.
  # No mutable-staging fallback or store open occurs when admission fails.
  # Legacy `resolve_read_cfg` can still select live staging for non-snapshot
  # callers; that does not make staging query-safe.
  # Flat publishing keeps its in-place read behavior.
  serve: "current"
  # SCAL-30: a dense query must visit EVERY shard, so shard count is query
  # cost. Two knobs decide it:
  #   shard_buckets — buckets per file type. 0 (default) keeps the historic
  #     one-per-first-character layout, up to 37 shards per type before any
  #     size rolling. Set 2-8 to cut fan-out proportionally; only affects
  #     where new writes route, existing shards stay readable.
  #   shard_max_bytes — roll to a new shard at this size (default 1 GiB).
  #     At scale this dominates: a 1.4 TB store rolls ~1400 shards at the
  #     default, and every query visits all of them. Raise it for large
  #     corpora; the trade is a bigger cold open per shard.
  # Changing either only pays off on a fresh store — existing shard files
  # keep their original routing.
  # A query whose `search.filter` pins `source` (a bare value, `$eq`, or
  # `$in`) visits only the shards of those files' types; any other filter
  # shape still fans out fully, because skipping a shard that might hold a
  # match would silently cut recall. When the fan-out does not finish
  # inside `search.fanout_timeout_s` the query returns what completed and
  # logs `shard_fanout_partial` — recall was cut for that query.
  # shard_buckets: 0
  # shard_max_bytes: 1073741824
  # PERF-83: HNSW index knobs, passed to a NEW doc collection only —
  # Chroma ignores collection metadata for one that already exists, so a
  # change here applies to new shards and to a --force-reindex, never to
  # a live collection. Every key is OPTIONAL and unset means "whatever
  # the installed Chroma chooses"; do not uncomment one to write down
  # today's default, or an upgrade's better default is silently
  # overridden. `chroma_add_chunks` is ~99.6% of main-process phase time
  # (ADR 0017), and these are the knobs that decide what one insert
  # costs.
  #   sync_threshold — records buffered before the graph is flushed to
  #     disk. No recall effect, and raising it MEASURED WORSE: +84.5%
  #     write cost at 50000 over 60k chunks (ADR 0017). Not a free win.
  #   batch_size — records per internal insert batch. No recall effect;
  #     measured -3.8% at 1000, within noise.
  #   m — graph degree. Measured -36.7% write cost and -30% query p50 at
  #     8, but a smaller graph trades recall. Re-score on the eval
  #     harness before adopting (PERF-84).
  #   construction_ef — candidate breadth while building. Measured
  #     -23.1% write cost at 50. Same recall caveat.
  #   search_ef — candidate breadth at query time. Higher = better
  #     recall, slower queries.
  # hnsw:
  #   sync_threshold: 1000
  #   batch_size: 100
  #   m: 16
  #   construction_ef: 100
  #   search_ef: 10
  retain_generations: 2
  # RS-SCALE-1: run the orphan-aux-row reclamation sweep on store open.
  # Normally indexer-managed (persist.py injects it for maintenance
  # opens) and left False on the query side so the first search doesn't
  # pay a whole-collection metadata scan. Set true only to restore the
  # query-side sweep. `codexa store gc` reclaims orphans on demand.
  # reclaim_orphans_on_open: false

# PROD-5 / ADR 0014: model-download policy. When true (default) a model
# that is missing from the local HF cache is fetched once, so a clean host
# can index without a separate bootstrap step. A warm cache never contacts
# the network — loads run under HF_HUB_OFFLINE=1 against
# `<persist_dir>/hf_cache`. Set false for an air-gapped install: a cache
# miss then raises an actionable error naming `codexa fetch embedding`
# instead of reaching the Hugging Face Hub. Corpus content, queries, and
# the index never leave the host under either setting.
allow_model_download: true

embeddings:
  # Registered embedder factory. "local" is the built-in offline
  # SentenceTransformers adapter. "llama_cpp_vulkan" runs a local
  # embedding-capable GGUF through Vulkan and fails closed if GPU offload
  # is unavailable; see docs/performance.md for its required runtime/build
  # dependencies. Plugins may register another name.
  backend: "local"
  # HF identifier OR a local on-disk path. With the identifier, pre-cache
  # once via `codexa fetch embedding`; subsequent loads run under
  # `HF_HUB_OFFLINE=1` against the cache pinned to
  # `<store.persist_dir>/hf_cache` (see codexa/utils/hf_cache.py).
  # Catalog E5 ids automatically apply their required `passage:` /
  # `query:` input recipe; changing that recipe rebuilds the index.
  # Point this at an absolute path (e.g. a restored snapshot directory)
  # for a strictly offline first run — no network at all.
  model_path: "sentence-transformers/all-MiniLM-L6-v2"
  # Vulkan GGUF example (requires backend: "llama_cpp_vulkan"):
  # model_path: "/srv/codexa/models/all-minilm-l6-v2-f16.gguf"
  # Immutable HF commit for the shipped MiniLM weights. Codexa applies this
  # default only while model_path remains the shipped id; changing model_path
  # alone clears the pin so it cannot be sent to an unrelated model. Set a
  # model-specific commit explicitly when selecting another remote HF model.
  # revision: "1110a243fdf4706b3f48f1d95db1a4f5529b4d41"
  # model_revision: "1110a243fdf4706b3f48f1d95db1a4f5529b4d41"  # legacy alias
  # Matryoshka embedding truncation target. Leave unset/null unless the
  # selected model advertises supported matryoshka dims in the catalog;
  # both index and query vectors are truncated to this dim.
  # matryoshka_target_dim: null
  # Upper bound for the encoder batch. Auto-batch will halve this under
  # RAM or VRAM pressure (CUDA only) and double back up once both calm
  # down. 32 matches the code default; push it higher only after
  # confirming the accelerator has headroom; OOM kills the whole run.
  # The Vulkan backend also uses this as its maximum sequences per call.
  batch_size: 32
  # SentenceTransformers device probe: "auto" picks cuda → mps → xpu → cpu in that
  # order. Override with "cuda", "mps", "xpu", "cpu", or a specific
  # index like "cuda:1". AMD ROCm uses the cuda branch automatically.
  # This knob does not select a llama.cpp Vulkan device; until ADAPT-8,
  # set GGML_VK_VISIBLE_DEVICES before starting Codexa.
  device: "auto"
  # Soft VRAM cap applied on CUDA: PyTorch's allocator won't grow past
  # this fraction of total device memory, so a re-index can't starve the
  # rest of the desktop of GPU memory.
  cuda_memory_fraction: 0.7
  # Auto-batch pressure heuristic (CFG-2). When `batch_size` halving under
  # memory pressure is enabled, these set the hysteresis band: shrink the
  # batch once RAM (or CUDA VRAM) usage crosses the `_high` fraction, grow
  # back only once it falls below the `_low` fraction. Fractions are in
  # (0, 1]; out-of-range / garbage values fall back to the defaults below.
  # Lower the `_high` knobs on constrained hardware to back off sooner.
  auto_batch: true
  # auto_batch_ram_high: 0.60     # shrink when RAM ≥ this fraction
  # auto_batch_ram_low: 0.40      # grow when RAM ≤ this fraction
  # auto_batch_vram_high: 0.80    # shrink when CUDA VRAM ≥ this fraction
  # auto_batch_vram_low: 0.55     # grow when CUDA VRAM ≤ this fraction
  # auto_batch_recheck_every: 8   # re-poll pressure every N batches (≥1)
  # Cap for BLAS / tokenizer / PyTorch intra-op threads in the main
  # process (where the embedder runs). Without this, every encode()
  # could fan out into cpu_count BLAS threads. Operator-set env vars
  # (OMP_NUM_THREADS=…) always win.
  #
  # On CPU-only hosts this is the dominant throughput knob — far more
  # impactful than `indexer.workers`. The ramp-up bench shows scaling
  # is near-linear up to ~8 threads, then sharply diminishing returns
  # (T=8→12 +13%, T=12→16 +3%). Default 8 = the empirical sweet spot
  # on a 32-core box; lower it on smaller hosts to leave headroom for
  # the rest of the desktop.
  main_thread_cap: 8
  cache:
    enabled: true
    dir: "./chroma_db/emb_cache"
    shards: 26
    # CACHE-20/SCAL-21: total on-disk cap for the embedding cache.
    # 0 (default) = uncapped; >0 prunes coldest entries after each
    # run. Change `shards` without losing the cache via
    # `codexa cache reshard --shards N` (a bare config bump re-hashes
    # every key → full re-embed). ~64-128 MB/shard is a good target.
    # max_bytes: 0

ocr:
  enabled: true
  engine: "paddle"      # fastest measured; "tesseract", or "both" (merged)
  # Paddle uses the GPU only when a CUDA/ROCm paddlepaddle build is
  # installed; the plain wheel is CPU-only.
  languages: "eng"
  # PERF-105: PaddleOCR 3.x builds a full document pipeline unless told
  # otherwise. Measured on identical scanned pages at 300 dpi, one
  # worker: shipped 49.2 s/page recovering 6,088 chars; without these
  # two stages 43.3 s and 6,843 chars; also on the mobile detector
  # 16.5 s and 6,548. Page-orientation classification and UVDoc
  # dewarping were not a speed/accuracy trade on upright, flat book
  # scans - they cost 13% of the run and removed 12% of the text. Turn
  # them on for photographed or skewed pages, which is what they exist
  # for. Textline orientation stays on either way: it measured free.
  paddle_doc_preprocessing: false
  # Detection model. PaddleOCR otherwise pairs a server-class detector
  # with a mobile recogniser; the mobile detector measured 2.98x faster
  # end-to-end while still recovering more text than that pairing.
  # "auto" defers to PaddleOCR's own choice.
  paddle_detection_model: PP-OCRv5_mobile_det
  pdf_dpi: 300          # Render DPI for scanned PDF pages (72-600).
  # PERF-102: measured over 400 corpus PDFs / 1,643 scanned pages, the
  # native resolutions cluster hard at 300 dpi. 300 renders 96.7% of
  # scans at or above their own resolution; 260 reached only 30.5%
  # because it falls in the gap below that cluster. Cost scales
  # ~DPI^2.7, so 400 buys 2 points of coverage for 2.2x the pixels.
  # Run `python scripts/analyze_pdf_dpi.py --config config.yml
  # --suggest-config` to get the value your own corpus wants.
  pdf_page_gate: image  # PERF-103: which pages are rasterised at all.
  # "image"       - the "chars" rule below, restricted to pages that
  #                 carry an image. Default. A captioned scan still
  #                 gets OCR'd; a text page with no image never does.
  # "page_shaped" - narrower: only text-free pages whose largest image
  #                 is shaped like the page. Cheapest; misses tiled
  #                 scans and captioned figures.
  # "chars"       - legacy: any page under pdf_text_threshold_chars,
  #                 which measured 73.6% of its work on pages whose
  #                 text was already extractable.
  # When a PDF's text-layer averages fewer than this many chars per page,
  # the extractor decides the PDF is image-only and falls back to inline
  # OCR. Lower = OCR less aggressively (trust the embedded text more);
  # higher = OCR more aggressively. Only consulted when ocr.enabled.
  pdf_text_threshold_chars: 512
  # When a CHM archive's extracted HTML averages fewer than this many
  # characters per page, OCR its referenced raster pages. Set 0 to disable
  # the CHM fallback while keeping ordinary image/PDF OCR enabled.
  chm_text_threshold_chars: 256
  # OCR embedded image objects inside PDF pages in addition to the page
  # raster. Useful for diagram/caption-heavy PDFs; default off because
  # it adds one OCR pass per embedded image.
  # in_image_extract: false
  # WATCH-4: per-engine OCR wall-clock budget in whole seconds. 0 (default)
  # disables it — each engine call runs unbounded. Set a positive value
  # (e.g. 120) so a pathological image can't wedge the OCR worker
  # indefinitely; on expiry the engine returns empty text + an
  # `ocr_engine_timeout` event fires.
  engine_timeout_s: 0
  # Bounded per-stream FlateDecode retry cap (MiB): 1-512, default 256.
  # A page that still exceeds the cap routes to OCR; without OCR it fails
  # with actionable remediation instead of risking multi-GiB allocation.
  pdf_decompress_limit_mb: 256
  # Content-addressed OCR cache. Keyed on sha256(file_bytes) + (engine,
  # languages, pdf_dpi), so a re-index of unchanged files under the
  # same OCR config is nearly free. Disabled by default — turn on for
  # OCR-heavy corpora where re-indexing under different configurations
  # is common, or when re-rasterizing every PDF page is the bottleneck.
  cache:
    enabled: false
    dir: "./chroma_db/ocr_cache"
    shards: 8
    # CACHE-23: total entry cap across shards. Oldest rows from the largest
    # shard are pruned every `prune_every_writes` successful writes and when
    # an OCR worker exits. Content + recipe fingerprints invalidate stale rows.
    max_entries: 100000
    prune_every_writes: 256

# SVG handling. The base extractor parses <text>/<tspan>/<textPath>/
# <title>/<desc> via stdlib XML — zero extra deps. SVGs that ship
# their text as path outlines (logo vectorisation, font-to-curves
# diagrams) or that embed raster <image href="data:..."> blobs come
# back empty from that pass. `render_fallback` rasterises the SVG and
# OCRs the bitmap when the XML pass is shorter than
# `text_threshold_chars`. Needs the [svg] extra (`pip install -e ".[svg]"`)
# plus the system Cairo runtime (`libcairo2` on Debian/Ubuntu; `cairo` on
# Homebrew); without them the fallback is a silent no-op. Only consulted
# when ocr.enabled.
svg:
  render_fallback: true
  render_dpi: 150          # SVG → bitmap render DPI for the OCR fallback.
  text_threshold_chars: 16 # Below this many XML chars, rasterise + OCR; 0 disables fallback.

# Image object detection. OCR pulls *words* off an image; this pulls
# *objects* — a photo of a kitchen with no on-image text used to
# index as "" and vanish. When enabled, _extract_image appends an
# `[objects: chair, table, …]` line to the OCR text. Independent of
# ocr.enabled: run either, both, or neither.
#
# Offline-first: the model is operator-supplied — no runtime fetch.
# Drop a YOLOv8-class `.onnx` at <store.persist_dir>/vision/model.onnx
# or point `model_path` elsewhere. Needs the [vision] extra
# (`pip install -e ".[vision]"` → onnxruntime, no torch). With the
# extra or the model file absent, detection is a silent no-op and
# OCR is unaffected. Stock YOLOv8 exports carry the COCO-80 class
# set; a custom model points `class_names_path` at a newline-
# delimited label file.
vision:
  enabled: false
  backend: "onnx"          # or a detector registered by a plugin
  model_path: ""           # "" → <store.persist_dir>/vision/model.onnx
  class_names_path: ""     # "" → bundled COCO-80
  input_size: 640          # ONNX model input square size (e.g. 416/512/640/1280).
  score_threshold: 0.35    # Drop below this confidence [0,1]; 0 accepts any positive score.
  max_labels: 12           # Cap distinct object labels per image.

llm:
  provider: "none"    # "ollama", "local", or "none"
  fallback: "local"     # tried when `provider` is unreachable; same allowed values
  # Ollama (external server)
  model: "llama3"
  host: "http://localhost:11434"
  # Intra-Ollama model chain. When `model` times out / errors, the
  # dispatcher retries each entry below before giving up on the
  # provider and falling over to `llm.fallback`. Order to taste:
  # smaller-first ("get any answer fast") or bigger-first ("escalate
  # for quality"). Empty list (or omitted) keeps single-model
  # behaviour. Each emits a one-line `ollama_model_failed` warning
  # on miss so the chain is visible in logs.
  model_fallbacks: []
  # Embedded llama.cpp provider — deterministic (temperature=0, top_k=1, fixed seed).
  # Install with `pip install -e ".[local-llm]"`, then point `model_path` at a GGUF.
  local:
    model_path: "/path/to/models/your-model.gguf"
    # 0 = unbounded: load the model's full trained context window
    # (`n_ctx_train`) so a long-context model isn't capped — avoids the
    # llama.cpp warning `n_ctx_seq (4096) < n_ctx_train (...)`. Pin a
    # smaller value to bound the KV-cache RAM on memory-tight hosts.
    n_ctx: 0
    seed: 1337
    max_tokens: 512
  max_summary_chars: 500  # Truncate the LLM-generated corpus summary at this many characters.
  # Per-Ollama-call timeout in seconds. Default 300 covers cold-start
  # model loads on CPU-only hosts (the first inference can spend
  # 30–90 s mapping the weights into RAM before any tokens emerge).
  # Drop it to 30–60 on a fast GPU box; raise it on slow disks.
  # Caller-explicit `timeout=` kwargs (tests / scripted usage) still
  # win. Garbage values (booleans, strings, negatives) silently fall
  # back to the default rather than crash the dispatcher.
  # timeout: 300
  # Fast availability probe timeout in seconds. This is only for the
  # `/api/tags` startup/session probe, not generation. Raise it for a
  # loaded remote Ollama host that answers slowly; explicit caller
  # timeouts still win. Garbage values fall back to 3.0.
  # probe_timeout: 3.0
  # PERF-10: probe primary and fallback providers in parallel during
  # cold UI startup. Off by default to preserve the serial probe's
  # quieter resource footprint.
  parallel_probe: false

# Search-time enrichment knobs.
search:
  # One wall-clock deadline for shard, decomposed-query, and rewrite A/B
  # fan-outs. Healthy branches survive a timed-out peer.
  # fanout_timeout_s: 30
  # How many passages a search returns (the final slice). Default 25 —
  # the prior 5 was tight, so a broad question only saw the five closest
  # chunks. Effective count is bounded by the candidate pool, so very
  # large top_k also wants a higher overfetch_max below.
  # top_k: 25
  # PERF-7: collapse fuse + rerank into a parallel stage on wide
  # candidate pools. Default false preserves the sequential pipeline.
  # parallel_fuse_rerank: false
  # RAG-6: run semantic near-dup after rerank. PRF stays before every
  # reranker; final dedup preserves their learned input rank.
  # dedup_after_rerank: false
  # PLUG-7: override the hybrid fusion registry method. Built-ins:
  # "rrf" and "weighted"; plugins can register additional names.
  # fusion:
  #   method: "rrf"
  # RAG-1/BOT-1: rewrite conversational follow-ups into standalone
  # queries. `ab_fuse` retrieves original + rewrite and RRF-fuses them.
  # query_rewrite:
  #   enabled: false
  #   ab_fuse: false
  # RAG-1: split compound questions into sub-queries before retrieval.
  # query_decomp:
  #   enabled: false
  # IR-8: expand child chunks to their parent section after rerank.
  # expand_to_parent:
  #   enabled: false
  # RETQ-9/CFG-23: live diversity metric for reranker-collapse triage.
  # metrics:
  #   runtime_diversity: false
  # IR-18/19/20: offline cross-lingual RAG. When on, the query is
  # translated (offline, via the configured LLM) into every language the
  # corpus carries (the manifest's per-file `language` labels), each
  # variant is retrieved, the pools are RRF-fused, and one answer is
  # composed over the multilingual pool. Default off — single-language
  # corpora pay nothing.
  multilingual:
    enabled: false
  # IR-21: offline synonym / alias query expansion. When on, the query is
  # fanned into variants via an operator-supplied YAML/JSON alias map
  # (`{NYC: ["New York", "New York City"]}`); each variant is retrieved and
  # RRF-fused with the original. Default off; needs `synonyms_path` set.
  query_expansion:
    enabled: false
    synonyms_path: ""    # path to the alias map; "" = no expansion
  # PROMPT-4 / CFG-4: optional persona / tone directive injected into the
  # RAG prompt. The body lands AFTER the safety + locale rules (so it can
  # never relax the citation / NO_ANSWER / language contracts) and is run
  # through the sentinel-escape + length cap, so a runaway value can't crowd
  # out the corpus context or smuggle a fence token. Bounded to 600 chars
  # (`_PERSONA_MAX_CHARS`); longer text is truncated. Empty / unset → no
  # persona slot (prompts stay byte-identical to the no-persona build).
  # persona:
  #   text: "Answer as a concise, plain-spoken technical writer."
  # Cap on the number of prior conversation turns fed back into the
  # RAG prompt's `Recent conversation history:` block. Default 6 =
  # 3 user + 3 assistant turns. Older turns survive on disk under
  # `sessions.json` but don't pay the context-window tax. Drop to
  # 0 to disable history injection entirely (useful on tight-context
  # models). Non-positive / non-integer values silently fall back
  # to the default.
  # history_turns: 6
  # When true, history turns dropped past `history_turns` get
  # condensed into a 2-3 sentence summary by the LLM and prepended
  # as a synthetic assistant turn. Keeps long threads coherent at
  # the cost of one extra LLM call per follow-up. Result is cached
  # on the session row, so a repeated query against an unchanged
  # thread reuses the prior summary. Off by default — opt-in only.
  # history_compress: false
  # CFG-5 (post-2026-05-28): `history_compress` accepts the new enum
  # form `"never" | "auto" | "always"` in addition to the legacy
  # bool. `"auto"` triggers compaction only when the live session's
  # character count breaches `history_max_chars`, leaving short
  # threads on the plain bounded-history path.
  # history_compress: "never"
  # COH-1: which compaction STRATEGY runs once `history_compress`
  # decides a follow-up should compact. "summary" (default) LLM-
  # summarises the dropped prefix into a synthetic recap turn;
  # "relevance" (BOT-9) keeps the recent tail verbatim + the top
  # `history_relevance_anchors` OLDER turns scored against the live
  # query, so an early topic-introducing turn the thread keeps
  # revisiting survives past `history_turns`.
  # history_compress_strategy: "summary"
  # history_relevance_anchors: 2
  # Maximum cached recap length before an over-budget summary is
  # summarized again. Valid range: 200-20000 characters.
  # history_summary_budget_chars: 1500
  # Token ceiling for each history-summary LLM call. Valid range: 32-2000.
  # history_summary_max_tokens: 200
  # CFG-8: include recent user turns in the dense retrieval query
  # embedding, not just in the final prompt. Off by default to keep
  # standalone queries literal; turn on for follow-up-heavy chats.
  # history_in_dense: false
  # history_in_dense_turns: 2  # 0 disables history query augmentation.
  # CFG-2: BOT-6 character budget for the prompt's history block.
  # Default 8000 chars ≈ 2000 tokens — comfortable on a 4k-ctx
  # model. Lift on long-context models. 0 disables the auto-compact
  # trigger (history_compress: auto becomes a no-op).
  # history_max_chars: 8000
  # CFG-2: BOT-10 per-assistant-turn smart-truncation cap. Default
  # 200 chars; 0 disables truncation. User turns stay verbatim.
  # history_assistant_max_chars: 200
  # CFG-2: RAG-3 passage-text budget for the `{context}` slot.
  # Default 8000 chars; raise on long-context models. 0 disables
  # trimming.
  # context_max_chars: 8000
  # Per-passage clamp before formatting. Defaults to context_max_chars;
  # set 0 to disable the per-row cap while keeping the total context budget.
  # per_passage_max_chars: 8000
  # Prompt context budget unit. "chars" preserves the legacy budget;
  # "tokens" uses the configured embedding model's tokenizer before
  # admitting another passage. context_max_chars supplies either limit.
  # context_budget_unit: "chars"
  # RET-9 / RETQ-33: relevance-gate mode for dense first-pass and PRF
  # expansion candidate pools; both passes use the same policy.
  # Disable only to measure/troubleshoot the horizon's contribution.
  # horizon_enabled: true
  #   percentile (default) — keep candidates whose distance is at or
  #     below `max_distance_percentile` of the pool; adapts to the
  #     embedder's own distance scale so a keyword query like "linux"
  #     no longer empties just because MiniLM puts short queries near
  #     the cosine midpoint.
  #   absolute — keep rows below the fixed `max_distance` cosine
  #     ceiling (legacy 1.0 behaviour, with the soft-floor fallback).
  # max_distance_mode: percentile
  # max_distance_percentile: 70   # percentile mode: keep closest ~70%
  #
  # CFG-7: maximum dense-vector distance accepted from the store, used
  # only when max_distance_mode is `absolute`. Chroma cosine distance
  # is 0 for identical vectors and grows as candidates get less
  # similar. Non-numeric / out-of-range values fall back to 1.0.
  # max_distance: 1.0
  # CFG-1: PERF-6 answer cache. Schema-versioned key covers the exact
  # rendered prompt, ordered evidence, and output-affecting LLM /
  # grounding policy; retries + save-restore can skip the LLM call.
  # Disabled by default — opt-in only.
  # answer_cache:
  #   enabled: false
  #   max_entries: 256  # 0 disables answer caching.
  #   ttl_s: 86400      # 0 keeps entries until eviction/reset.
  # CFG-3: IR-10 HyDE — generate `n` hypothetical answer paragraphs
  # at search time and embed each as an additional sub-query. The
  # extra LLM calls are bounded by `n`; default 1 keeps the
  # pre-IR-10 cost unchanged. Clamped to [1, 5].
  # hyde:
  #   enabled: false
  #   n: 1
  # CFG-7: search-insights abstractive flag. The extractive (LexRank)
  # path is governed by `insights.enabled`; setting `abstractive:
  # true` runs a configured LLM summarisation on top, cached on
  # `(source, hash)`.
  # insights:
  #   enabled: false
  #   abstractive: false
  #   max_sentences: 3
  #   top_keyphrases: 8
  # CFG-4: RAG-2 grounding-score baseline. Per `_calibrate_grounding`
  # the normalised score is `(raw - baseline) / (1 - baseline)`,
  # clamped to [0, 1]. Set 0.0 to disable normalisation entirely
  # (raw scores ride through to the citation chips).
  # citations:
  #   verify: false          # render the citation-grounding expander
  #   threshold: 0.3         # flag sentences below this grounded score
  #   baseline_score: 0.3
  # Faithfulness UI/replay controls. `show_internal_external_split: null`
  # keeps auto mode: show the split only when external rows are present.
  # true forces it on; false suppresses it even with external rows.
  # faith:
  #   show_blocking_flags: true
  #   show_internal_external_split: null
  #   replay_score: false
  # RAG-7 one-shot revision pass for unsupported sentences. Off by
  # default; when enabled, the dispatcher regenerates once if any
  # sentence scores below `threshold`.
  # regenerate_when_ungrounded:
  #   enabled: false
  #   threshold: 0.3
  # DPI for on-demand PDF page thumbnails rendered from a retrieved
  # passage's source. 100 (default) is a balance between legibility
  # and cost; 200+ takes seconds per page on CPU. Non-positive /
  # non-integer values silently fall back to the default.
  # passage_thumbnail_dpi: 100
  # Cross-encoder reranker over the retrieved top-K. Disabled by
  # default — the cross-encoder costs ~50-200 ms per query on CPU
  # (sub-50 ms on GPU) but reranks pairs end-to-end, catching
  # nuance the bi-encoder misses (negation, multi-word entities,
  # query intent). Worth turning on when answer quality matters
  # more than latency. The model auto-downloads on first use
  # (~80 MB for `BAAI/bge-reranker-base`); subsequent runs serve
  # from the HF cache. Failure modes (no network, broken install)
  # log once and fall back to bi-encoder ranking.
  # reranker:
  #   enabled: false
  #   backend: "sentence_transformers"  # or a registered plugin backend
  #   model_id: "BAAI/bge-reranker-base"
  #   top_k_keep: 5
  #   device: "auto"  # "auto" → cuda when available, else cpu
  #   min_score: 0.0   # Drop cross-encoder rows below this sigmoid floor.
  # Optional learned-sparse reranker over the fused candidate pool.
  # Disabled when `model_id` is empty. When set, Codexa lazily loads a
  # SentenceTransformers SparseEncoder and sorts rows by sparse dot
  # product before the cross-encoder reranker. `top_k_keep: 0` keeps
  # the full pool for later stages; set a positive cap to prune.
  # sparse:
  #   backend: "sentence_transformers"  # or a registered plugin backend
  #   model_id: ""
  #   device: "auto"  # "auto" → cuda when available, else cpu
  #   top_k_keep: 0
  # IR-4 ColBERT late-interaction rerank tier (needs the `[colbert]` extra).
  # IR-22 `precompute_corpus: true`: cache each candidate chunk's
  # ColBERT doc vectors once per process (keyed by chunk doc-id) instead
  # of re-encoding them every query. IR-23 `precompute_corpus: "persist"`
  # writes an index-time SQLite sidecar that survives restarts; enable only
  # when the extra index time + disk footprint are acceptable. Default off.
  # colbert:
  #   enabled: false
  #   backend: "pylate"
  #   model_id: "lightonai/Reason-ModernColBERT"
  #   device: "auto"
  #   top_k_keep: 5
  #   precompute_corpus: false
  # A/B answer comparison in the Streamlit search panel. Off by default;
  # when enabled the panel can ask two configured providers side by side.
  # compare:
  #   enabled: false
  #   provider_a: "ollama"
  #   provider_b: "local"
  # Streaming grounding chips score each completed streamed sentence
  # against retrieved passages. The non-streaming answer path still scores
  # final text when this is off.
  # streaming_grounding:
  #   enabled: false
  #   threshold: 0.3
  # Follow-up suggestions under each answer. Extractive mode is local;
  # `use_llm: true` asks the configured LLM for richer suggestions.
  # follow_ups:
  #   enabled: false
  #   use_llm: false
  # CFG-1: WIRE-15 quality-floor stage. Drops rows whose IDX-side
  # `quality_score` falls below the floor BEFORE the cross-encoder
  # rerank pays to rank them. Default 0.0 = no-op (legacy behaviour).
  # Operators with noisy corpora (OCR garbage, boilerplate-heavy
  # crawl, autogenerated PDFs) can opt in at e.g. 0.2 to prune the
  # bottom decile. Clamped to [0.0, 1.0].
  # quality_floor: 0.0
  # CFG-1: RETQ-4 candidate-pool over-fetch caps. Base controls how
  # many rows the dense + sparse passes pull before fuse + rerank;
  # `_reranker` widens it when the cross-encoder is on so the
  # reranker has more rows to choose from. Defaults are tuned for
  # ≤100k-file corpora; large operators can widen to lift recall@5
  # at the cost of one extra round-trip per query.
  # overfetch_max: 40
  # overfetch_max_reranker: 60
  # CFG-1: WIRE-14 FLARE length-normalised confidence floor. Triggers
  # passage refresh when the LLM's mean-token logprob over the
  # current draft falls below this threshold. Null/absent disables
  # the length-normalised path; the legacy single-token threshold
  # at `flare.min_logprob` remains independent.
  # flare:
  #   # Confidence strategy: auto (logprob then locale-aware lexical),
  #   # logprob, lexical, or a plugin-registered detector name.
  #   detector: auto
  #   # WIRE-29 single-token logprob confidence floor. Unset (null) →
  #   # a per-model calibrated default; set an explicit value to override.
  #   min_logprob: null
  #   mean_logprob_threshold: null
  #   # WIRE-47: default true buffers each attempt; a low-confidence
  #   # retrigger discards that draft and re-invokes the LLM over the
  #   # augmented passage set. False keeps one progressive attempt inside
  #   # FLARE; the outer grounding gate still emits only its completed answer.
  #   restart_on_retrigger: true
  #   # Re-trigger controls. `max_rounds` caps FLARE re-trigger passes
  #   # (0 disables re-triggering); `top_k_extra` is how many extra
  #   # passages each round pulls; `max_passages` is the ROB-16 ceiling
  #   # on the augmented passage set per round.
  #   max_rounds: 3
  #   top_k_extra: 5
  #   max_passages: 60
  # Embedding near-duplicate dedup, layered after lexical MMR.
  # Catches *paraphrase* duplicates (same
  # meaning, different words → low token overlap but near-identical
  # vectors) the Jaccard filter structurally can't see. Embeds the
  # already-deduped ≤30 rows once and drops any within `threshold`
  # cosine of an earlier-kept row. ON by default — set
  # `enabled: false` to opt out (saves one embed pass per query).
  # semantic_dedup:
  #   # Lexical MMR runs both before and after PRF; disable separately
  #   # for evaluation/troubleshooting without disabling vector dedup.
  #   mmr_enabled: true
  #   enabled: true
  #   threshold: 0.93   # cosine ≥ this ⇒ near-duplicate (0 < t ≤ 1)
  #   # ADAPT-1: lexical Jaccard near-dup drop (kills
  #   # byte-level adjacent-chunk overlap before the embedding pass) +
  #   # the empty-pane soft floor (how many closest rows to keep when
  #   # the distance horizon would drop everything).
  #   jaccard_threshold: 0.85   # token-overlap ≥ this ⇒ duplicate
  #   mmr_lambda: 0.7           # 1=relevance first, 0=diversity first
  #   soft_floor_top_n: 3
  # Pseudo-relevance feedback (Rocchio query expansion). Assumes the
  # top survivors are relevant, pulls the query vector toward their
  # centroid, and re-queries. Recovers on-topic passages that shared
  # few surface terms with the original query. Merges the two result
  # sets so PRF only ever ADDS recall — it can't drop a first-pass
  # hit. ON by default — costs one extra embed + store query per
  # search; set `enabled: false` to opt out on latency-tight hosts.
  # prf:
  #   enabled: true
  #   top_m: 5          # how many top survivors feed the centroid
  #   alpha: 1.0        # original-query weight
  #   beta: 0.75        # feedback-centroid weight
  # LLM-free passage insights under the results: an extractive
  # LexRank TL;DR + KeyBERT key phrases, computed with the search
  # embedder already in memory (no provider, no network). OFF by
  # default — opt in for an at-a-glance semantic layer even when no
  # LLM is configured. Cheap: bounded to the lead ~8 passages /
  # 6000 chars.
  # insights:
  #   enabled: false
  #   max_sentences: 3   # extractive summary length
  #   top_keyphrases: 8  # key-phrase chips shown
  # Bold recognised entities inline inside each retrieved passage
  # (NLTK NER, already loaded for the entity chips). OFF by default
  # — switches the passage render from st.write to markdown; the
  # rest of the text is markdown-escaped so it can't mis-render.
  # highlight_entities: false
  # Hybrid dense+sparse retrieval. `enabled` turns on in-memory BM25
  # over dense candidates; `use_corpus_bm25` also queries the persisted
  # corpus BM25 index during search. `persist_bm25` writes that sparse
  # index during indexing; it defaults to `enabled` when omitted.
  hybrid:
    # RETQ-44: on by default. BM25 re-ranks the dense candidates, which
    # over 465 gold queries from a real corpus gained keyword NDCG@10
    # +0.075 / MRR@10 +0.082 and verbatim +0.047 / +0.051 (paired 95%
    # CIs clear of zero). Nothing is written at index time for it.
    enabled: true
    # Also query the persisted sparse index when one exists. Worth far
    # more than the in-memory pass — keyword NDCG@10 +0.248, MRR@10
    # +0.236, recall +0.277 — and with no sidecar on disk it returns
    # exactly the in-memory result, so it is safe to leave on.
    use_corpus_bm25: true
    # Write that sidecar during indexing. OFF by default despite the
    # numbers above: SCAL-42 stores the full path-derived doc-id on
    # every posting row, so it costs 42.8 KB per chunk (a 2.12 GB
    # bm25.db for 48k chunks, 73% of the generation) and projects to
    # 0.28-2.29 TB on a 214k-file corpus. Turn it on deliberately, on a
    # corpus whose size you have checked.
    persist_bm25: false
    mode: "rrf"             # "rrf" or "weighted"
    rrf_k: 60               # RRF rank constant; higher smooths rank gaps.
    alpha: 0.5              # Weighted mode dense-vs-sparse blend.
    quality_weight_enabled: false  # Opt-in quality_score discount.
    # Optional eval-harness YAML/JSON overlay. `codexa eval tune-hybrid
    # --apply` writes an authoritative overlay for enabled/mode/rrf_k/alpha;
    # ordinary hand-written overlay files only fill missing keys.
    tuned_params_path: ""
    # WIRE-18 corpus-BM25 fetch cap (bounds the sparse candidate pass).
    # 0 / unset → the dense-pool size at query time; a positive value
    # caps the sparse fetch explicitly.
    # bm25_top_k: 0
  # ADAPT-1: BM25 scoring parameters for the in-memory hybrid rerank.
  # `k1` = term-frequency saturation (higher → repeated terms keep
  # adding weight); `b` = length normalisation in [0,1] (1.0 = full
  # normalisation by document length). Defaults match rank_bm25
  # (k1=1.5, b=0.75); grid-tunable like search.hybrid.{rrf_k,alpha}.
  # bm25:
  #   k1: 1.5
  #   b: 0.75
  wikipedia:
    # Offline encyclopedic background for RAG answers. The dispatcher
    # looks every query up against a local index built from:
    #   1. the bundled seed in `codexa/data/wikipedia_seed.json`
    #      (~25 common topics, CC BY-SA 4.0), and
    #   2. any extra JSON files in `data_dirs` (same schema).
    # No network calls — codexa stays offline-first. The prompt
    # template explicitly attributes any included summary to
    # Wikipedia + CC BY-SA 4.0, so the model never silently
    # passes encyclopedic context off as primary corpus material.
    # Default-on because the lookup costs nothing when the index
    # doesn't match the query.
    enabled: true
    # WIRE-30: the offline Wikipedia fallback is surfaced as a result
    # row whenever the corpus has no usable match — an empty pool, OR a
    # 'not indexed' query whose closest passage is past this cosine
    # distance. Default 0.6: with all-MiniLM-L6-v2 an on-topic passage
    # lands under ~0.6, while off-topic noise (e.g. this corpus answering
    # a Linux question with its closest ~0.88 hit) is judged 'no match'
    # and gets the fallback. Raise toward 1.0+ to fall back less often.
    # fallback_max_distance: 0.6
    # ML-34: non-cosine stores use their calibrated [0,1] similarity
    # instead of comparing an L2/IP distance with the cosine-only knob.
    # fallback_min_similarity: 0.6
    # Maximum plain-text characters retained from a matched ZIM article.
    # zim_summary_max_chars: 1500  # 200..20000
    # RETQ-42: least share of the question's content words a ZIM title
    # must explain before its article is accepted. The suggestion index
    # answers one salient term, so at 0.0 "how does a router forward
    # packets" resolves to the dotfile article `.forward` and that text
    # reaches the LLM as grounded context. Lower to enrich more often
    # and less accurately.
    # zim_min_title_coverage: 0.6  # 0.0..1.0
    # SCAL-39: how many archives one query may probe. The walk is
    # serial, sits on the search path, and each candidate pays an
    # archive open unless `zim_max_open` covers the walk. A 327-archive
    # library took 64.8 s on a query nothing answered before this cap.
    # zim_max_archives: 12  # 1..10000
    # Extra JSON files / directories with operator-supplied
    # entries. Each file follows {license, license_url, source,
    # entries: [{title, extract, url}, ...]}. Drop a fresh file in
    # one of these dirs to extend coverage without rebuilding the
    # package.
    data_dirs: []
    # Kiwix ZIM archive(s). When configured, queries that miss the
    # bundled seed run a full-text search against every archive via
    # `libzim` (`pip install -e ".[zim]"`). Stays offline.
    #
    # IMPORTANT — this is the *search-time* Wikipedia tier, NOT
    # corpus ingestion. Two distinct uses for ZIMs in Codexa:
    #
    #   A. `cfg.search.wikipedia.zim_path` (this knob)
    #      → on-demand full-text lookup AT SEARCH TIME.
    #      → renders as `📚 Wikipedia: <title>` in the RAG prompt.
    #      → zero indexing cost; the ZIM stays untouched on disk.
    #      → use when you want cheap encyclopedic background
    #        grounding for every query.
    #
    #   B. Drop a `.zim` file under `data_dirs` (top-level cfg)
    #      → full corpus ingestion: every article walked, chunked,
    #        embedded, and queryable as regular passages.
    #      → expensive (hours of embed time on full English ZIM).
    #      → use when you want the ZIM contents to BE part of the
    #        local corpus rather than a side-channel reference.
    #
    # The two are independent. You can wire a ZIM to BOTH knobs (cheap
    # background lookups + persistent corpus inclusion), to either one
    # alone, or to neither.
    #
    # Three shapes are supported on this knob (combine freely):
    #   1. Single file:  zim_path: "/path/to/wikipedia_en.zim"
    #   2. List of files (multi-language blending):
    #        zim_path:
    #          - "/path/to/wikipedia_en.zim"
    #          - "/path/to/wikipedia_pt.zim"
    #   3. Directory auto-discovery (every *.zim file inside is opened):
    #        zim_dir: "/path/to/wiki/"
    #
    # The dispatcher walks every resolved archive in resolution order
    # and returns the first hit, so an English query that misses a pt
    # ZIM still surfaces the en match without manual cfg switching.
    # Bulk-download every supported language with
    # `codexa-fetch-zim --all-langs` (or `codexa-check-deps
    # --fetch-zims`); both default to `{persist_dir}/wiki/` and write
    # `zim_dir` here automatically.
    # Download individual archives from https://library.kiwix.org/.
    # Hybrid resolution: `zim_path` pins the Wikimedia-family archives
    # in preference order (pt_BR locale → pt first, then en variants,
    # then es/fr/it/de; sister projects last), so encyclopedic queries
    # hit Wikipedia / Wiktionary / Wikibooks / Wikiquote / Wikisource /
    # Wikiversity / Wikivoyage before the alphabetical fallback.
    # `zim_dir` then auto-discovers every other *.zim in the dir
    # (Stack Exchange dumps, TED talks, Gutenberg, Khan Academy,
    # NHS/MedlinePlus, cheatsheets, etc.) as the secondary tier.
    # First-hit-wins across the merged list.
    # Example (opt-in): pin Wikimedia-family archives in preference order,
    # then `zim_dir` auto-discovers the rest. Empty by default.
    #   zim_path:
    #     - "/path/to/wiki/wikipedia_en_all_nopic_YYYY-MM.zim"
    zim_path: []
    zim_dir: ""
    # CFG-56: maximum warm ZIM archive handles per process. Default 12,
    # matching `zim_max_archives` — one query walks that many
    # candidates, and a smaller cap makes the walk evict handles it is
    # about to reopen. It was 1: measured on a 327-archive library,
    # 12.65 s per query at a cap of 1 against 0.15 s once the walk fits.
    #
    # A warm handle costs the dirent/Xapian header working set, not the
    # archive: the body is mmap'd, so twelve handles are address space
    # plus reclaimable page cache, not twelve archives resident.
    #
    # Tune it for your environment. Raising `zim_max_archives` without
    # raising this leaves the walk evicting itself again, so that case
    # is warned about once; setting it lower is honoured for hosts under
    # real memory pressure.
    zim_max_open: 12
    # Online Wikipedia REST API as the *last* fallback (opt-in;
    # default false because it breaks the offline-first guarantee).
    # Only consulted when both the seed and the ZIM (if configured)
    # miss. Useful on workstations with patchy connectivity that
    # want a final safety net.
    online: false
    # Timeout and retry ceiling for the opt-in online tier. Kept short so
    # a flaky network degrades back to corpus-only instead of stalling RAG.
    online_timeout_s: 3.0
    online_max_attempts: 3
    # Wikipedia language code for the ZIM and online tiers — `auto`
    # derives from the active i18n locale (`pt_BR` → `pt`); set
    # explicitly to `en`, `pt`, `es`, etc. to override.
    lang: "auto"

# OBS-53: per-concern sidecar logs, written beside indexer.jsonl with the
# same rotation + redaction policy:
#   health.jsonl        — one snapshot per interval (phase, progress, rate,
#                         ETA, RSS). Written by a timer, so a wedged phase
#                         still produces a pulse.
#   skipped_files.jsonl — deliberately not indexed (encrypted/unreadable/empty)
#   failed_files.jsonl  — should have indexed and did not
#   slow_files.jsonl    — files slow enough to be worth attention
logging:
  health_interval_s: 300

metadata:
  index_info_file: "./chroma_db/index_info.json"
  # Manifest file. Use a `.json` suffix for the human-readable backend
  # (default), or `.db` / `.sqlite` for the SQLite backend — recommended
  # for corpora >50k files since saves become per-row writes inside a
  # single transaction instead of an O(n) full-file rewrite.
  # SQLite is the new-install default; explicit `.json` paths remain valid.
  manifest_file: "./chroma_db/manifest.db"
  # Saved-chats store. Default `{store.persist_dir}/sessions.json`
  # (JSON backend, full-file rewrite per save). Switch to a `.db` /
  # `.sqlite` suffix for the SQLite backend — row-level UPSERTs
  # in a single transaction per save, recommended once the
  # operator accumulates >~1k threads. Backend dispatch uses the
  # path suffix; no other knob. On platforms without POSIX fcntl
  # locks, `.json` session paths route to a sibling `.db` automatically.
  # sessions_file: "./chroma_db/sessions.db"
  # Pilot evaluation run ledger. Relative paths resolve from this config file;
  # when unset, the runtime uses `{store.persist_dir}/pilot_runs.jsonl`.
  # pilot_runs_file: "./chroma_db/pilot_runs.jsonl"
  # Latest saved eval-baseline summary consumed by the Chatbot Ops
  # dashboard. Write it with:
  #   codexa eval baseline --save ./chroma_db/eval_baseline_summary.json
  # Default when unset: `{store.persist_dir}/eval_baseline_summary.json`, then
  # the checkout's `.codexa-quality/eval_baseline_summary.json` CI artifact.
  # eval_summary_file: "./chroma_db/eval_baseline_summary.json"
  # Optional entity-linking KB (YAML/JSON) for alias → canonical
  # resolution before entity storage. Empty/unset disables linking.
  # knowledge_base_path: ""
  enable_entity_extraction: true
  tag_count: 50
  # Entity-extraction backend + optional linguistic transformations.
  # Both toggles are off by default and must be written as unquoted
  # booleans — a quoted "false" is rejected by `codexa validate-config`
  # rather than silently switching the transformation on.
  # entities:
  #   backend: "nltk"        # nltk (default) | spacy | stanza
  #   # CL-2: substitute pronouns with their referent before extraction.
  #   # Needs the [entities] extra (fastcoref or coreferee); slow.
  #   coref: false
  #   # CL-9: collapse each span onto its lemma so running/runs/ran
  #   # count as one entity.
  #   lemmatize: false

indexer:
  # IDX-12: `codexa index --daemon` poll interval in seconds — how often
  # the daemon re-scans data_dirs + indexes the deltas (the IndexerLock is
  # released between passes so the UI keeps serving queries). Floored at 1s.
  # `--watch-interval` overrides this per-run; 30 is a sane default.
  watch_interval_s: 30
  # Persistent pass failures back off exponentially from watch_interval_s
  # to this ceiling. Success resets the next delay to the normal interval.
  daemon_max_backoff_s: 900
  # Safely interrupted generation builds retain their frozen plan and
  # checkpoints for this long (30 days). Expired or incompatible plans are
  # never resumed. This is a disk-retention window, not a safety gate: a
  # resume rehashes every source in the frozen plan first, so a corpus that
  # drifted while the build sat is caught however old the plan is. Sized for
  # a first build measured in weeks — a shorter window strands the work an
  # interrupted long build had already finished.
  paused_generation_ttl_s: 2592000
  # Opt-in live KPI publication after a successful index pass. The cases
  # file is operator-owned; every expected source must exist inside one of
  # `data_dirs`, otherwise evaluation is skipped and no summary is written.
  # post_index_eval:
  #   enabled: false
  #   cases_path: "./eval_cases.yml"
  #   # WIRE-51: retrieval targets recorded with the summary so the
  #   # chatbot-ops scoreboard scores target-vs-actual. Each is
  #   # optional; an unset one records no target row. These describe
  #   # the run — unlike `codexa eval baseline --min-*` they never
  #   # drive an exit code, because a published index stays usable.
  #   min_recall: 0.60
  #   min_precision: 0.30
  #   min_ndcg_at_5: 0.50
  # Requested chunking workers (ProcessPoolExecutor). The default
  # ceiling stays at the measured-safe 8. A guarded 16-worker trial must
  # explicitly raise `chunk_worker_ceiling` and provide the measured peak
  # RSS of one representative worker; otherwise admission retreats to 8.
  # CPU admission reserves the embedder thread cap plus one logical core,
  # never exceeds physical cores, and RAM admission projects workers under
  # 70% of total host memory. Every limit appears in `effective_budget`.
  #
  # On CPU-only hosts the chunker is rarely the bottleneck — the
  # embedder is. Tune `embeddings.main_thread_cap` and
  # `embeddings.batch_size` to push throughput.
  workers: 4
  # Maximum admitted chunk workers. Valid range: 1..16; 8 is the safe
  # default. Use `scripts/bench_indexer.py --worker-trial-16` to obtain a
  # matched 8-vs-16 decision before opting in.
  #
  # Raising this alone may change nothing: the effective count is the
  # minimum of this ceiling, the CPU limit (which never exceeds physical
  # cores) and the RAM limit, and the RAM limit retreats to 8 unless
  # `chunk_worker_rss_mb` is set. Read `effective_budget` in the run log
  # to see which limit actually bound — on a 32-logical/16-physical host
  # it reported `ceiling=8 cpu_limit=16 ram_limit=8`, so the ceiling and
  # the unmeasured-RSS retreat were binding together and the CPU had
  # twice the headroom being used.
  # chunk_worker_ceiling: 16
  # Peak RSS in MiB measured for one representative chunk worker. A zero
  # or missing value forbids admission above 8. The matched trial carries
  # its 8-worker measurement into the candidate run automatically.
  # Err high: RAM admission divides the host budget by this number, so
  # an over-estimate admits fewer workers while an under-estimate can
  # over-commit memory. Measured 1122-1926 MiB per worker on an EPUB
  # batch (2026-08-15).
  # chunk_worker_rss_mb: 1024
  # Temporary disk budget for completed chunk results held behind an ordered
  # head straggler. Zero keeps the proven in-memory backpressure path.
  # The drain stops submitting once `workers * 8` finished results are
  # waiting on the blocked head, so on a corpus with a heavy tail one slow
  # file idles the rest of its lane: measured 2026-08-15, a PDF lane fell
  # from 4 busy workers to 1 within four minutes of a 14-minute head file
  # and stayed there with 19,366 files still queued. Spilling the blocked
  # tail releases the window. PERF-89's benchmark measured this 2.3% slower,
  # but none of its trials had a long-blocked head — it paid for a spool it
  # did not need. Enable when your corpus has that tail; leave it at zero
  # when file cost is uniform. Valid range: 0..4096 MiB.
  # chunk_result_spill_mb: 0
  # Cross-file embed batching thresholds. The streaming indexer merges
  # chunks from small files into one embed() call until either threshold
  # fires, then slices vectors back per file before storing.
  embed_buffer_files: 8
  embed_buffer_chunks: 256
  # Abort after this many consecutive persistent vector-store write
  # failures instead of spending the rest of the run on a wedged store.
  store_failure_breaker_threshold: 5
  # OCR is even heavier (paddle/tesseract models per process); capped to
  # 2 in practice. Same downscale-warning rule as `workers`. The OCR
  # ramp-up bench shows 1→2 doubles per-file OCR throughput, with no
  # meaningful CPU/RAM penalty on a workstation-class host.
  ocr_workers: 2
  # ROB-116 progressive publication starts with a small source batch so
  # a clean rebuild becomes searchable early, then doubles each batch up
  # to the maximum to bound generation-clone and publication overhead.
  # Values above 65536 fall back to these safe defaults.
  # These bound automatic progressive publication for eligible clean generation builds.
  # progressive_first_batch_size: 256
  # progressive_max_batch_size: 4096
  # Stop admitting later checkpoints after this many seconds. A checkpoint
  # already in progress finishes atomically before the driver returns.
  # progressive_run_budget_s: 3600
  skip_dirs: []         # Extra directory names to skip (in addition to defaults like .git, node_modules)
  # Optional scan scope. These match `codexa index --include`,
  # `--exclude`, and `--max-files`; CLI flags override them per run.
  include_globs: []     # Keep only absolute paths matching any glob.
  exclude_globs: []     # Drop paths matching any glob after include.
  max_files: 0          # 0 = no cap; useful for ramp-up tests.
  # DIST-1: change detection skips re-hashing a known file when its
  # (mtime_ns, size) still match the manifest. mtime+size are forgeable:
  # a backup restore / `rsync -a` / `cp -p` preserves them, so content can
  # change while the stat is identical and the file is silently kept as
  # "unchanged" — drifting the index from the corpus. Set this to false on
  # corpora maintained that way to force a content re-hash of every known
  # file each run (slower, but exact). Default true keeps the fast path.
  trust_mtime: true
  # RS-OBS-2: one-shot OCR-lane maintenance flag. Prefer the CLI
  # `codexa index --only-ocr`; this config knob exists for scheduled
  # runs that need to reprocess image/CBZ OCR rows without resetting the
  # vector store. Do not combine with force_reindex.
  only_ocr: false
  # Rebuild the index from scratch. Prefer `codexa index --force-reindex`
  # for one-off runs; this config knob exists for scheduled rebuild jobs.
  # Do not combine with only_ocr.
  force_reindex: false
  # Preview the planned index changes without writing the store,
  # manifest, or index metadata. Prefer `codexa index --dry-run` for
  # one-off checks; this config knob exists for scheduled audits.
  dry_run: false
  # RS-RELDEP-2: copy the current store + manifest/index-info before a
  # destructive `--force-reindex`. Off by default because large Chroma
  # stores can be hundreds of GB; enable before risky migrations. Empty
  # backup root means sibling `<persist_dir>.backups`; Codexa adds a
  # stable corpus namespace before each `reindex-...` directory.
  backup_before_reindex: false
  reindex_backup_dir: ""
  # RS2-ADAPT-1/ROB-14: bounded wait (seconds) when the indexer epilogue
  # closes a manifest sqlite connection whose lock is still held by a
  # wedged borrower. On timeout the entry is force-evicted (gen bump) and
  # its close deferred. Default 5.0; raise it for slow/contended
  # shared-FS stores. Floored at 0.1s.
  manifest_lock_timeout_s: 5.0
  # WATCH-2/MT-2/WATCH-16: abandon a single chunk future that runs longer than this
  # many seconds (logged `chunk_worker_timeout`, the file recorded as an
  # error) so one wedged file — a pathological PDF parser loop or a hung
  # tokenizer — can't stall the whole indexing drain. The 1800-second
  # default is a total per-file extract/chunk deadline; set 0 only when an
  # unbounded file is intentional. The wedged worker is reaped at teardown.
  chunk_file_timeout_s: 1800
  # Replace a poisoned pool this many consecutive times without a successful
  # file completion. Useful progress resets the streak; 0 disables recycling.
  # WATCH-28: a new chunk-worker pool gets this long to run one trivial
  # task before the lane gives up on it. A pool that cannot start a
  # worker never will, and the lane blocks inside its generator while it
  # waits - so its terminal sentinel is never published and the only
  # backstop is the consumer's stall watchdog an hour later. 0 restores
  # the unbounded wait.
  chunk_pool_warmup_timeout_s: 60.0
  chunk_pool_max_restarts: 3
  # WATCH-16: defer unchanged timeout inputs for one day instead of paying
  # the full deadline every daemon pass. `--retry-failed` bypasses once.
  chunk_timeout_retry_after_s: 86400
  # Quarantine unchanged input bytes after this many chunk timeouts.
  # 0 preserves the legacy retry-forever behaviour.
  chunk_timeout_max_attempts: 3
  # WATCH-1/WATCH-3: log `indexer_stalled` / `lane_stalled` (WARNING) when
  # embed progress / a producer lane makes no progress for this many
  # seconds — turns a silent hang into a visible signal instead of
  # progress lines that just stop. 0 disables it. Set it above
  # your slowest single file's embed time (a long CPU encode legitimately
  # freezes the counter) to avoid false alarms — e.g. 1800 = 30 min of
  # zero progress.
  stall_timeout_s: 1800
  # Fail a producer-lane drain or an unchanged daemon child after two
  # consecutive effective stall windows (~62 minutes by default). The daemon reaps
  # the child's whole process group. Set 0 for warn-only; embed_flush_stalled
  # remains warn-only inside a non-daemon run because a live native call
  # cannot be interrupted safely in-process.
  stall_fail_after_intervals: 2
  # RS2-ADAPT-2: shutdown-grace (seconds) for joining the stall-watchdog
  # thread on context teardown. The thread is a daemon woken by a stop
  # event, so this only bounds the brief teardown wait. Default 2.0.
  watchdog_join_timeout_s: 2.0
  # Manifest checkpoint cadence during the embed loop. Saves happen
  # every N files OR every M seconds, whichever comes first. Increase
  # both on slow disks / huge corpora to cut SQLite write pressure;
  # decrease for snappier UI updates on small corpora.
  checkpoint_every_files: 10
  checkpoint_every_seconds: 5.0
  # HEART-1: how often (in files) the indexer emits a progress event for
  # the UI/logs. Floored at 1. Lower for snappier progress on small
  # corpora; raise to cut log volume on huge runs.
  progress_every_files: 100
  # IDX-11: PDF layout-aware extraction (reconstruct columns/tables).
  # Off by default — slower; enable for multi-column / tabular PDFs.
  pdf:
    layout_aware: false
  # CFG-8: IDX-7 orphan-chunk GC. At indexer epilogue, enumerate the
  # store's live `source` ids and delete any that don't appear in the
  # manifest. Off by default — first-run safety: pointing the
  # indexer at a partial manifest with this enabled would lose the
  # prior corpus' chunks. Enable once the manifest is canonical.
  # gc_orphan_chunks: false
  # Generate an optional LLM summary from the top corpus entities after a
  # work-producing pass. Off by default because a large local model can add
  # minutes to the indexing critical path.
  # corpus_summary_enabled: false
  # Append typed table/image/structured chunks onto extractor text
  # before embedding. Default off; malformed structured output logs
  # and the original text still indexes.
  # structured_extract:
  #   enabled: false
  # Rebuild chunks as an L0/L1/L2 hierarchy with stable node/parent IDs so
  # search.expand_to_parent can substitute broader sections. Default off;
  # changing the toggle rebuilds populated indexes. Legacy manifests rebuild
  # once to stamp the separate hierarchy producer identity.
  # hierarchical:
  #   enabled: false

# CFG-6: CL-7 tokenizer mode for `semantic/phrases.tokenize_for_indexing`.
#   "unicode"   — default; ASCII-word regex (legacy behaviour).
#   "url-aware" — preserves `https?://…` as a single token.
#   "cjk-aware" — emits overlapping bigrams across CJK runs +
#                 url-aware behaviour. ASCII runs in mixed text stay
#                 on the default tokenizer.
# Unknown values fall back to "unicode" so a typo can't silently
# rewrite the index's token shape.
# semantic:
#   tokenizer: "unicode"

# Token-level chunking parameters. `chunk_size` is the per-chunk cap;
# overlap is computed dynamically per file (10% of token count, min 20).
#
# Cross-file paragraph dedup (IDX-5/CL-11): drop paragraphs already
# emitted for a different source in the same indexer run (license
# headers, EPUB nav blocks, stub banners). Off by default.
#   cross_file_dedup:        false  # enable the literal-fingerprint pass
#   cross_file_near_dedup:   0.0    # Jaccard floor for the near-dup pass
#                                   # (0 disables; ~0.85 = "OCR variant")
#   cross_file_shingle_size: 3      # word-shingle width for near-dup
# SCAL-17: cap on retained dedup state (literal fingerprints +
# near-dup shingle sets). Once a run tracks more than this many
# paragraphs the oldest entries are evicted LRU-style so a 10M-
# paragraph corpus can't pin 1-2 GB. 0 = unbounded (legacy behaviour).
#   cross_file_dedup_cap:    200000
# DS-1: when `datasketch` (MinHash-LSH backend, `[dedup]` extra) is
# absent, the near-dup gate is a linear shingle scan — this caps it to
# the most-recent N paragraphs (sliding window) so the fallback stays
# O(N*window) instead of O(N^2). 0 = unbounded scan (legacy).
#   cross_file_near_dedup_fallback_cap: 2000
chunking:
  chunk_size: 512
  # CFG-25 / WIRE-4 / RETQ-40: prose splitter strategy. "recursive"
  # (the default) prefers paragraph/sentence boundaries; "window" is the
  # legacy sliding token window, which cut 80.3% of chunks mid-sentence
  # against 9.2% for recursive over 60 documents at chunk_size 512.
  # Changing this changes chunk identity, so it invalidates an existing
  # index — re-index with --force-reindex after switching.
  # splitter: "recursive"
  # WIRE-30 / IDX-8: when true, source files whose extension maps to a
  # supported grammar (.py/.js/.ts/.go/.java/…) are split at function /
  # class / method boundaries via the code-aware chunker instead of the
  # prose sliding window, so a function body stays in one passage.
  # tree-sitter (the `[code]` extra) gives accurate splits; without it a
  # regex indent-aware fallback runs. Off by default — enabling it
  # changes how code files chunk, so re-index to apply.
  # code_aware: false

# File format allow/deny lists. Leave both empty (or omit) to index every
# supported format. `enabled` is a whitelist; `disabled` always wins.
# Examples:
#   enabled: [".pdf", ".epub"]   # only these
#   disabled: [".chm"]            # everything except .chm
#
# ZIM archives (`.zim`) are picked up by the libzim-backed extractor —
# ingestion walks every C-namespace article in the archive. Needs the
# `[zim]` extra (`pip install -e ".[zim]"`); without libzim installed
# the extractor returns an empty string and the file is logged but
# skipped, so the corpus build won't crash on a missing dependency.
#
# `text_fallback` controls what happens when no registered extractor
# matches the file's extension. True (default) reads the bytes as UTF-8
# with errors='ignore' for genuinely unknown text-like formats. Known
# opaque-binary suffixes (`BINARY_SKIP_EXTS`: archives/media/databases/
# executables such as .zip, .gz, .mp3, .db) always return "" first and
# log `binary_skipped`, even when this fallback remains true. Add a
# registered extractor plugin when archive-member ingestion is required.
# Set `text_fallback: false` only when every unknown extension should
# be dropped instead of decoded.
# Even with `text_fallback: true`, the fallback read is capped at
# `text_fallback_max_mb` (default 64): a multi-GB binary / garbage
# file under data_dirs is decoded to its prefix only (logged
# `text_fallback_truncated`) so it can't OOM the indexer or
# detonate the chunker. Raise it if you genuinely index >64 MB
# plain-text/log files; 0 falls back to the default.
# FMT-10 binary guard (defense-in-depth, independent of `text_fallback`).
# Even with the fallback ON, `binary_guard` (default true) sniffs the
# leading bytes before decoding and short-circuits to "" — logging
# `binary_guard_tripped` — when the content looks binary: a NUL byte
# (decisive; text files effectively never carry one, and BOM-led
# UTF-16/32 is exempted) or a control-byte density above
# `binary_guard_ctrl_ratio` (default 0.30; \t\n\r\f + all high bytes
# >=0x80 count as text so accented Latin-1 / UTF-8 prose never trips;
# 0 flags any non-NUL control-byte presence).
# `binary_guard_sample_bytes` (default 8192) bounds how many leading
# bytes are inspected; 0 inspects the whole capped read. This catches
# any unknown-extension binary — including a *future* format with no
# registered extractor yet — that `BINARY_SKIP_EXTS` does not enumerate.
# Set `binary_guard: false` to restore the pre-FMT-10 decode for
# non-denied suffixes; `BINARY_SKIP_EXTS` still wins.
formats:
  enabled: []
  disabled: []
  text_fallback: true
  text_fallback_max_mb: 64
  binary_guard: true
  binary_guard_sample_bytes: 8192
  binary_guard_ctrl_ratio: 0.30
  # DjVuLibre subprocess budgets. `djvutxt` handles embedded text layers;
  # `ddjvu` renders pages for OCR fallback when no text layer exists.
  djvu_txt_timeout_s: 120
  djvu_render_timeout_s: 300

# Plugin runtime knobs. The whole section is optional — leave it out
# and codexa uses defaults. Per-plugin config slices live as siblings
# of `disabled` / `executor_workers` / `slow_threshold_ms`, e.g.:
#   plugins:
#     my_filter:
#       timeout_s: 0.5
#       # Required security filters can fail closed when their entry point
#       # cannot load far enough for Codexa to inspect the plugin kind.
#       # fail_closed: true
plugins:
  # RS-SEC-2: deny-by-default allowlist. Any distribution installed in
  # the venv that registers the `codexa.plugins` entry-point group is
  # ignored UNLESS its entry-point ID is listed here — so a
  # typosquatted / supply-chain plugin can't run code at load just by
  # being importable. Runtime Plugin.name values are not accepted because
  # Codexa cannot know them without importing the plugin. Find IDs with
  # `codexa plugins list`. Empty / unset → no third-party plugins load.
  enabled: []          # e.g. ["word_counter"] to opt the sample plugin in
  # Entry-point IDs in `disabled` override `enabled` (kill switch).
  # disabled: ["word_counter"]
  # Cap on the timeout-dispatch thread pool; only matters when non-UI
  # plugins opt into a `timeout_s`. UI panels stay on Streamlit's script
  # thread and use timeout_s as an elapsed-warning threshold. Default is 4.
  executor_workers: 4
  # slow_threshold_ms: 1000

# Local-only product analytics. Disabled by default; when enabled,
# Diagnostics reads persisted session metadata and displays aggregate
# quality rates. No network call or export is performed.
analytics:
  local_quality_rollup: false

# DDD-6: Optional ChatbotObjective metadata for the operational
# dashboard (OBS-14 / BOT-24 / BOT-25). Pin the assistant's
# declared objective + success metrics + owner so the dashboard
# has an anchor for "is this chatbot doing its job." Codexa is
# personal-RAG by default; leave the section out → no objective
# is declared + the dashboard surfaces stay quiet.
# chatbot:
#   objective:
#     objective: "Answer questions about the Cetacean Stranding Network corpus."
#     success_metrics:
#       - name: "retrieval@5"
#         target: 0.70
#       - name: "no-answer rate"
#         operator: "<="
#         target: 0.10
#     owner: "marine-ops-team"
#     review_interval_days: 30
#     set_at: "2026-04-01T00:00:00Z"