diff --git a/.dockerignore b/.dockerignore
index 1f05501..c5d9e9b 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -52,6 +52,11 @@ README*
CHANGELOG*
TODO*
+# …except README.md: pyproject.toml declares `readme = "README.md"`, so the
+# build backend needs it present to install the project. Excluding it made
+# `uv sync` in the Dockerfiles die with "Readme file does not exist".
+!README.md
+
# Development and build tools
Makefile
docker-compose*.yml
diff --git a/Dockerfile b/Dockerfile
index a03fd56..dbc52b6 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -16,23 +16,39 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
curl \
&& rm -rf /var/lib/apt/lists/*
-# Install uv
-COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+# Install uv from PyPI at a pinned version.
+#
+# This used to be `COPY --from=ghcr.io/astral-sh/uv:latest`, which pulled
+# an unpinned tag: image contents changed under you between builds, and a
+# breaking uv release could break the build with no diff to show for it.
+# PyPI is already required by every other layer here, so sourcing uv from
+# it also drops a second registry from the build's dependency set.
+ARG UV_VERSION=0.8.17
+RUN pip install --no-cache-dir "uv==${UV_VERSION}"
# Set working directory
WORKDIR /app
-# Copy dependency files first for cache efficiency
-COPY pyproject.toml uv.lock ./
+# Copy dependency files first for cache efficiency.
+# README.md is required: pyproject.toml declares it as the project readme,
+# so the build backend fails without it when uv installs the project below.
+COPY pyproject.toml uv.lock README.md ./
# Install dependencies (without the project itself)
+# Extras matter here: a bare `uv sync` installs only the core dependencies, so
+# the image shipped without sqlalchemy, langgraph, prometheus-client or pyjwt —
+# /metrics served nothing, DATABASE_URL failed with "No module named
+# 'sqlalchemy'", and JWT auth could not be enabled at all. `all` restores those
+# and matches what `agentomatic deploy` builds. `db-postgres` is named
+# separately because `all` deliberately carries only the SQLite driver, and
+# this image is a deployment: the compose stack beside it offers Postgres.
RUN --mount=type=cache,target=/root/.cache/uv \
- uv sync --frozen --no-install-project --no-dev
+ uv sync --frozen --no-install-project --no-dev --extra all --extra db-postgres
# Copy source code and install the project
COPY src/ ./src/
RUN --mount=type=cache,target=/root/.cache/uv \
- uv sync --frozen --no-dev
+ uv sync --frozen --no-dev --extra all --extra db-postgres
# Production stage
FROM python:3.12-slim
diff --git a/Dockerfile.distroless b/Dockerfile.distroless
index abc22d7..8976475 100644
--- a/Dockerfile.distroless
+++ b/Dockerfile.distroless
@@ -3,13 +3,17 @@
# built-in ``nonroot`` account (numeric UID 65532) so images honour
# Kubernetes ``runAsNonRoot`` admission policies out of the box.
-# Build stage
-FROM python:3.12-slim AS builder
+# ---- Build stage ------------------------------------------------------------
+# Python 3.11 on purpose: ``distroless/python3-debian12`` ships Debian 12's
+# Python 3.11, and dependencies must be built for the interpreter that will
+# actually import them. Building on 3.12 produced an image that could not start
+# at all — the venv's ``bin/python`` symlinked to the builder's 3.12 binary,
+# which does not exist in the runtime stage, so the ENTRYPOINT was a dangling
+# symlink; and even resolved, cp312 wheels cannot be imported under 3.11.
+FROM python:3.11-slim AS builder
-# Set environment variables
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
- UV_COMPILE_BYTECODE=1 \
UV_LINK_MODE=copy
# Install system dependencies
@@ -18,40 +22,40 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
curl \
&& rm -rf /var/lib/apt/lists/*
-# Install uv
-COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+# Install uv from PyPI at a pinned version. An unpinned `uv:latest` image made
+# builds irreproducible, and PyPI is already required by every other layer.
+ARG UV_VERSION=0.8.17
+RUN pip install --no-cache-dir "uv==${UV_VERSION}"
-# Set working directory
WORKDIR /app
-# Copy dependency files
-COPY pyproject.toml uv.lock ./
-
-# Install dependencies
-RUN --mount=type=cache,target=/root/.cache/uv \
- uv sync --frozen --no-install-project --no-dev
-
-# Copy source code and install
+# README.md is required: pyproject.toml declares it as the project readme, so
+# installing the project below fails without it.
+COPY pyproject.toml uv.lock README.md ./
COPY src/ ./src/
-RUN --mount=type=cache,target=/root/.cache/uv \
- uv sync --frozen --no-dev
-# Ensure the runtime user (uid 65532 / nonroot) can read the venv and sources
-RUN chown -R 65532:65532 /app
+# ``--target`` instead of a virtualenv: the runtime stage runs the distroless
+# image's own interpreter, which cannot use a venv built around a different
+# Python binary. A plain directory on ``PYTHONPATH`` works with any 3.11.
+#
+# Extras matter here: a bare install ships core dependencies only, leaving the
+# image without sqlalchemy, langgraph, prometheus-client or pyjwt — /metrics
+# served nothing, DATABASE_URL failed with "No module named 'sqlalchemy'", and
+# JWT auth could not be enabled. ``db-postgres`` is named separately because
+# ``all`` deliberately carries only the SQLite driver.
+RUN --mount=type=cache,target=/root/.cache/uv \
+ uv pip install --target=/app/deps ".[all,db-postgres]"
-# Production stage — distroless, non-root by default
+# ---- Runtime stage ----------------------------------------------------------
FROM gcr.io/distroless/python3-debian12:nonroot
-# Set environment variables
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
- PATH="/app/.venv/bin:$PATH" \
- VIRTUAL_ENV="/app/.venv"
+ PYTHONPATH="/app/deps"
WORKDIR /app
-# Copy virtual environment and application from builder
-COPY --from=builder --chown=65532:65532 /app/.venv /app/.venv
+COPY --from=builder --chown=65532:65532 /app/deps /app/deps
COPY --from=builder --chown=65532:65532 /app/src /app/src
COPY --from=builder --chown=65532:65532 /app/pyproject.toml /app/
@@ -59,17 +63,16 @@ COPY --from=builder --chown=65532:65532 /app/pyproject.toml /app/
# have to introspect the base image at admit-time.
USER 65532:65532
-# Expose port
EXPOSE 8000
-# Distroless has no shell, but it can execute binaries directly. We
-# invoke the ``agentomatic`` console script (installed by uv into
-# ``/app/.venv/bin``) via the venv Python so both the plain and
-# distroless images boot through the same ``agentomatic run`` entrypoint.
-ENTRYPOINT ["/app/.venv/bin/python", "/app/.venv/bin/agentomatic"]
+# Distroless has no shell, so run the CLI as a module through the base image's
+# own interpreter; dependencies come from PYTHONPATH. Invoking the console
+# script directly would go through its shebang, which points at the builder's
+# interpreter and is not present here.
+ENTRYPOINT ["/usr/bin/python3", "-m", "agentomatic.cli.commands"]
CMD ["run", "--agents-dir", "agents", "--host", "0.0.0.0", "--port", "8000"]
-# No shell and no curl in this image — hit /health with the venv Python
+# No shell and no curl in this image — hit /health with the base interpreter
# instead (exec form, so no shell is needed to run this CMD either).
HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \
- CMD ["/app/.venv/bin/python", "-c", "import urllib.request as u; u.urlopen('http://localhost:8000/health', timeout=5)"]
+ CMD ["/usr/bin/python3", "-c", "import urllib.request as u; u.urlopen('http://localhost:8000/health', timeout=5)"]
diff --git a/docker-compose.yml b/docker-compose.yml
index ddfabc7..5738603 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,52 +1,81 @@
+# =============================================================================
+# Agentomatic — local production-shaped stack
+#
+# docker compose up --build # platform on http://localhost:8000
+# docker compose --profile db up -d # …with Postgres-backed persistence
+#
+# One platform process serves every agent it discovers under ./agents — that
+# is the point of the product, so there is no per-agent service here. Features
+# are driven entirely by AGENTOMATIC_* env vars, exactly as in the image that
+# `agentomatic deploy` generates, so what you exercise locally is what ships.
+# =============================================================================
+
services:
- # Alpha Agent
- alpha-agent:
- build: .
- container_name: alpha-agent
+ platform:
+ build:
+ context: .
+ dockerfile: Dockerfile
+ image: agentomatic:latest
+ container_name: agentomatic-platform
+ restart: unless-stopped
ports:
- - "8001:8000"
+ - "${AGENTOMATIC_PORT:-8000}:8000"
environment:
- - AGENT_NAME=alpha
- - PORT=8000
- - OPENAI_API_KEY=${OPENAI_API_KEY}
- - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}
- command: ["agentomatic", "run", "--agents-dir", "agents", "--host", "0.0.0.0", "--port", "8000"]
+ AGENTOMATIC_TITLE: "${AGENTOMATIC_TITLE:-Agentomatic Platform}"
+ AGENTOMATIC_LOG_LEVEL: "${AGENTOMATIC_LOG_LEVEL:-INFO}"
+ # Studio, docs, health and metrics are on by default.
+ AGENTOMATIC_ENABLE_STUDIO: "${AGENTOMATIC_ENABLE_STUDIO:-1}"
+ AGENTOMATIC_ENABLE_METRICS: "${AGENTOMATIC_ENABLE_METRICS:-1}"
+ AGENTOMATIC_ENABLE_CONTROL_PLANE: "${AGENTOMATIC_ENABLE_CONTROL_PLANE:-1}"
+ # Opt-in hardening — set these in .env before exposing the port.
+ AGENTOMATIC_ENABLE_AUTH: "${AGENTOMATIC_ENABLE_AUTH:-0}"
+ AGENTOMATIC_API_KEY: "${AGENTOMATIC_API_KEY:-}"
+ AGENTOMATIC_CONTROL_TOKEN: "${AGENTOMATIC_CONTROL_TOKEN:-}"
+ AGENTOMATIC_ENABLE_RATE_LIMIT: "${AGENTOMATIC_ENABLE_RATE_LIMIT:-0}"
+ # Durable invocation history. Empty DATABASE_URL keeps it in-memory.
+ AGENTOMATIC_LOGS_HISTORY: "${AGENTOMATIC_LOGS_HISTORY:-0}"
+ DATABASE_URL: "${DATABASE_URL:-}"
+ # Bring your own model provider; nothing vendor-specific is baked in.
+ OPENAI_API_KEY: "${OPENAI_API_KEY:-}"
+ ANTHROPIC_API_KEY: "${ANTHROPIC_API_KEY:-}"
volumes:
- - ./agents:/app/agents
- networks:
- - agentomatic-network
+ # Drop agents in and restart — no rebuild needed for iteration.
+ - ./agents:/app/agents:ro
+ - agentomatic-data:/app/data
+ healthcheck:
+ test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
+ interval: 30s
+ timeout: 10s
+ retries: 3
+ start_period: 30s
+ networks: [agentomatic]
- # Beta Agent
- beta-agent:
- build: .
- container_name: beta-agent
- ports:
- - "8002:8000"
+ # Enable with: docker compose --profile db up -d
+ # then point the platform at it by setting, in .env:
+ # DATABASE_URL=postgresql+asyncpg://agentomatic:agentomatic@db:5432/agentomatic
+ # AGENTOMATIC_LOGS_HISTORY=1
+ db:
+ profiles: ["db"]
+ image: postgres:16-alpine
+ container_name: agentomatic-db
+ restart: unless-stopped
environment:
- - AGENT_NAME=beta
- - PORT=8000
- - OPENAI_API_KEY=${OPENAI_API_KEY}
- - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}
- command: ["agentomatic", "run", "--agents-dir", "agents", "--host", "0.0.0.0", "--port", "8000"]
+ POSTGRES_USER: "${POSTGRES_USER:-agentomatic}"
+ POSTGRES_PASSWORD: "${POSTGRES_PASSWORD:-agentomatic}"
+ POSTGRES_DB: "${POSTGRES_DB:-agentomatic}"
volumes:
- - ./agents:/app/agents
- networks:
- - agentomatic-network
+ - agentomatic-db:/var/lib/postgresql/data
+ healthcheck:
+ test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-agentomatic}"]
+ interval: 5s
+ timeout: 5s
+ retries: 20
+ networks: [agentomatic]
- # Nginx reverse proxy
- nginx:
- image: nginx:alpine
- container_name: agentomatic-proxy
- ports:
- - "80:80"
- volumes:
- - ./nginx.conf:/etc/nginx/nginx.conf:ro
- depends_on:
- - alpha-agent
- - beta-agent
- networks:
- - agentomatic-network
+volumes:
+ agentomatic-data:
+ agentomatic-db:
networks:
- agentomatic-network:
+ agentomatic:
driver: bridge
diff --git a/docs/FRONTEND_API_GUIDE.md b/docs/FRONTEND_API_GUIDE.md
index 4b1cf13..b96baf5 100644
--- a/docs/FRONTEND_API_GUIDE.md
+++ b/docs/FRONTEND_API_GUIDE.md
@@ -213,7 +213,7 @@ the frontend submits work, then **polls** or **streams** progress.
**Submit** (returns immediately with `202` and a task record):
- Agent: `POST /api/v1/{agent}/invoke/async` (single) · `/invoke/batch` (many)
-- Plugin: `POST /api/v1/{plugin}/predict/async` · `/predict/batch`
+- Plugin: `POST /api/v1/plugins/{plugin}/predict/async` · `/predict/batch`
- Pipeline: `POST /api/v1/pipelines/{name}/run/async` · `/run/batch`
- Endpoint: `POST /api/v1/endpoints/{name}{path}/async` · `.../batch`
- Ingestor: `POST /api/v1/ingestion/{name}/run/async` · `/run/batch`
diff --git a/docs/guide/deployment.md b/docs/guide/deployment.md
index 8405b39..14f5fd2 100644
--- a/docs/guide/deployment.md
+++ b/docs/guide/deployment.md
@@ -53,6 +53,50 @@ Everything above is configured declaratively — agents in folders, connections
in `connections.py`, endpoints in `endpoint.py`, and a handful of platform
flags. No custom FastAPI wiring is required.
+## Project dependencies in the generated image
+
+`agentomatic deploy` builds with **uv**, pinned via `ARG UV_VERSION` so the
+image is reproducible, and installs your project's own dependencies as well
+as agentomatic itself.
+
+Declare what *your* project needs — a vendor LLM driver, a vector client, an
+in-house package — in `requirements.txt` next to `main.py`:
+
+```txt title="requirements.txt"
+agentomatic[all]==1.10.0
+# This project's agents talk to a local OpenAI-compatible model server,
+# so it needs that client library.
+langchain-openai>=0.3
+```
+
+The generated Dockerfile installs agentomatic first (at its pinned version)
+and your requirements second, so a looser pin in your file cannot downgrade
+the framework.
+
+!!! warning "This is how a provider driver reaches the image"
+
+ The `all` extra deliberately excludes the vendor LLM drivers — `openai`,
+ `azure`, `vertex` — because the platform is provider-agnostic: you install
+ the SDK for the backend you actually use. `requirements.txt` is where that
+ happens. A stack configured for `openai_compatible` (which is what oMLX,
+ llama.cpp, vLLM and LM Studio all speak) needs `langchain-openai` there,
+ or the platform will refuse to start rather than answer with a fake model.
+
+### Reproducible builds with a lockfile
+
+If your project keeps a `pyproject.toml` and a `uv.lock`, the generated
+Dockerfile installs from the lock instead:
+
+```dockerfile
+COPY pyproject.toml uv.lock ./
+RUN uv sync --frozen --no-dev --inexact
+```
+
+That installs the exact resolved versions the lock pins rather than
+re-resolving at build time. Regenerate the lock with `uv lock` whenever you
+change a dependency.
+
+
## 1. Install for production
Install only the extras you use. Common production combinations:
diff --git a/docs/guide/optimization.md b/docs/guide/optimization.md
index 042f544..79e4be8 100644
--- a/docs/guide/optimization.md
+++ b/docs/guide/optimization.md
@@ -6,6 +6,22 @@ Inspired by Stanford's [DSPy](https://github.com/stanfordnlp/dspy), the framewor
---
+## Seeing the whole loop run
+
+`scripts/keras_showcase.py` runs `compile() → fit() → evaluate() → save() →
+load()` against any OpenAI-compatible endpoint and prints the measured loss
+curve, so you can watch the loop move before wiring it to your own agent:
+
+```bash
+export OMLX_BASE_URL=http://127.0.0.1:8000/v1
+export OMLX_API_KEY=whatever
+python scripts/keras_showcase.py --model omlx/my-local-model
+```
+
+Its agent answers correctly only once the prompt contains a token it has to
+*discover from its own failures*, so an improvement in the curve is
+attributable to the optimizer rather than to model variance.
+
## 🏗️ The Optimization Flow
The optimization loop coordinates datasets, rewriter LLMs, evaluator LLMs, and scoring metrics to iteratively improve prompt versions:
@@ -14,6 +30,44 @@ The optimization loop coordinates datasets, rewriter LLMs, evaluator LLMs, and s
---
+## Running the optimization suites without a cloud key
+
+The live optimization suites drive a real OpenAI-compatible endpoint. Point
+them at whatever local model you run — oMLX, llama.cpp, vLLM, LM Studio,
+Ollama — and they need no changes:
+
+```bash
+export OMLX_BASE_URL=http://127.0.0.1:8000/v1
+export OMLX_API_KEY=your-key
+export AGENTOMATIC_LIVE_MODEL=omlx/your-model
+
+uv run pytest tests/test_live_omlx_optimize.py \
+ tests/test_live_omlx_keras_optimize.py \
+ -q --override-ini='addopts='
+```
+
+Without such an endpoint these suites **skip entirely**, which leaves the
+`omlx/` provider path, the prompt fitter and the whole Keras-style `fit()`
+loop unexercised — including in CI. For that case the repo ships a stand-in:
+
+```bash
+uv run python scripts/local_slm_server.py --port 8000
+```
+
+`scripts/local_slm_server.py` is a **test double, not a language model**. It
+generates nothing; it follows rules. What makes it a valid optimization target
+is that answer quality genuinely depends on the system prompt — each directive
+a prompt carries makes the response satisfy one more property the metric
+rewards, so an optimizer that really searches and selects will climb, and one
+that does not will not. It also plays the rewriter (reading the briefing's
+failing I/O and expected answers, then folding the missing tokens into a new
+prompt) and the judge (returning the exact schema the metric asked for).
+
+It proves the *machinery* — search, evaluation, selection, early stopping,
+checkpointing, config application. It cannot tell you whether a real model
+writes good prompts. Use your own model and eval set for that.
+
+
## ⚡ Quick Start — two tiers (same primitives)
Agentomatic exposes **both** a thin one-shot path and a Keras-like staged path.
@@ -261,6 +315,18 @@ Agentomatic supports standard matches, LLM judges, and full **DeepEval** validat
- **Exact Match** (`exact_match`): Verifies if the agent response matches the expected answer exactly.
- **Contains** (`contains`): Verifies if the agent response contains a set of defined target keywords.
+!!! note "Matching metrics read the answer, not the whole reference"
+ An `AgentExample` with a structured `expected_output` is rendered for the
+ optimizer as a *judge-facing reference* — judge guidance, a rubric, an
+ `## Expected answer` section, the structured output as JSON. An LLM judge
+ reads all of it.
+
+ A matching metric compares strings, so it reads only the
+ `## Expected answer` section. Without that it would be comparing your
+ agent's response against markdown headers, and every candidate would score
+ near zero however good it was — `fit()` would report "no improvement"
+ forever. Plain-string expectations are used exactly as written.
+
### 2. LLM-as-a-Judge Metrics
- **LLM Judge** (`llm_judge`): Asks an evaluator LLM to grade the response on a scale of 0 to 1 based on custom criteria instructions.
- **G-Eval** (`g_eval`): Uses the G-Eval framework protocol to evaluate complex criteria (e.g. coherence, readability) with detailed scoring rubrics.
diff --git a/docs/guide/pipelines.md b/docs/guide/pipelines.md
index 434f0e0..9f0236e 100644
--- a/docs/guide/pipelines.md
+++ b/docs/guide/pipelines.md
@@ -303,6 +303,24 @@ steps:
`max`, `min`, `sum`, `sorted`, `isinstance`.
The `ctx` variable is a `PipelineContext` instance.
+!!! danger "A broken condition fails the step — it does not skip it"
+ A condition that *raises* (a typo, a renamed step, `$.` mapping syntax
+ where a `ctx` expression belongs) is a defect in the pipeline, not a
+ routing decision. The step is marked **failed** and the pipeline's
+ `on_error` policy decides what happens next.
+
+ ```yaml
+ # ❌ `$.` is mapping syntax — invalid Python, so this step fails.
+ condition: "$.classify.confidence < 0.7"
+
+ # ✅ A `ctx` expression.
+ condition: "ctx.get_step_output('classify').get('confidence', 0) < 0.7"
+ ```
+
+ `validate()` compiles every condition before a run, so a syntax error
+ like the first line above is rejected at load time (HTTP 422) rather
+ than at step three of a long pipeline.
+
### Loop
Repeat a step until a condition is met or a maximum iteration count is
diff --git a/docs/guide/platform-features.md b/docs/guide/platform-features.md
index 2f3f33e..25ed32e 100644
--- a/docs/guide/platform-features.md
+++ b/docs/guide/platform-features.md
@@ -484,10 +484,36 @@ By default the chain advances only on configured triggers (`timeout`,
Agentomatic provides **automatic conversation memory** for all deployed agents. When a thread store is configured, every `/chat` and `/invoke` call automatically:
1. **Loads prior conversation history** into the agent's `messages` state
-2. **Invokes the agent** with full conversational context
+2. **Invokes the agent** with that state
3. **Persists** both user and assistant messages to the store
4. **Summarises** older messages when the conversation grows long
+!!! warning "Your agent has to *read* `messages` — loading it is not enough"
+ The platform fills `state["messages"]` and reports `history_loaded`.
+ Whether the model ever sees those turns is the agent's decision. An
+ agent that sends only `current_query` answers every turn as if it were
+ the first, while the response still says `history_loaded: 12`.
+
+ A conversational agent should pass the turns through:
+
+ ```python
+ from agentomatic.langchain_adapter import dict_to_messages
+ from langchain_core.messages import SystemMessage
+
+ def respond(self, state: ChatState) -> ChatState:
+ # state.messages already ends with the current turn -- do not
+ # append state.request again, or the model sees it twice.
+ turns = dict_to_messages(
+ state.messages if state.messages else {"current_query": state.request}
+ )
+ result = self.llm.invoke([SystemMessage(content=self.prompt), *turns])
+ ...
+ ```
+
+ The `chatbot` and `langchain` templates ship this wiring. The other
+ templates take `current_query` alone on purpose: an extraction or
+ routing agent that dragged in prior turns would be the surprise.
+
```
Frontend Agentomatic Store
│ │ │
@@ -597,7 +623,7 @@ The response includes all agent output fields plus conversation metadata:
| `steps_taken` | Processing steps the agent took |
| `context` | Context data returned by agent (RAG docs, search results, etc.) |
| `metadata` | Merged metadata (request + agent + prompt_version) |
-| `history_loaded` | Number of prior messages loaded into context |
+| `history_loaded` | Prior messages **loaded from the store** — not proof the agent sent them to the model (see the warning above) |
| `duration_ms` | Processing time in milliseconds |
### Windowing & Summarization
diff --git a/docs/guide/verifying-a-deployment.md b/docs/guide/verifying-a-deployment.md
new file mode 100644
index 0000000..81ff237
--- /dev/null
+++ b/docs/guide/verifying-a-deployment.md
@@ -0,0 +1,102 @@
+# Verifying a Deployment
+
+`scripts/e2e_verify.py` drives every surface the platform publishes against a
+**running server** and reports pass/fail per group. It is deployment-agnostic:
+point it at `agentomatic run`, at `uvicorn main:app`, or at a container built
+by `agentomatic deploy`, and it adapts to what that deployment actually has
+switched on.
+
+Use it to answer the question a test suite cannot: *does this container, with
+this configuration, behave correctly right now?*
+
+```bash
+python scripts/e2e_verify.py \
+ --base-url http://localhost:8000 \
+ --agent my_agent --plugin my_plugin --pipeline my_pipeline \
+ --endpoint my_endpoint --ingestor my_ingestor \
+ --api-key "$AGENTOMATIC_API_KEY" \
+ --control-token "$AGENTOMATIC_CONTROL_TOKEN" \
+ --expect-auth \
+ --json report.json
+```
+
+The exit code is `0` only when every check passes, so it drops straight into
+CI or a post-deploy gate.
+
+## What it checks
+
+| Group | Covers |
+|---|---|
+| `platform` | `/health`, `/ready`, `/readiness`, `/status`, `/api/v1/status`, OpenAPI, Swagger, ReDoc, agent registry |
+| `studio` | Every call the bundled Studio React client makes — info, agents, graph, schemas, config, runs, thread state/history, the SSE run stream, and the SPA bundle itself |
+| `agent-rest` | `invoke`, `chat`, `invoke/stream` (SSE), `invoke/batch`, health, card, config, prompts, and the full thread lifecycle including fork, messages, summary, lineage, approvals and feedback |
+| `a2a` | Agent-to-Agent task submit, poll and cancel |
+| `plugins` | Registry, model card, health, `predict`, `predict/batch`, reload |
+| `endpoints` | Registry, info, health, call |
+| `ingestion` | Both `/api/v1/ingestion` and the `/api/v1/ingestors` alias the Studio bundle uses, plus per-ingestor info and health |
+| `pipelines` | Registry, config, validate, visualize, run, and `validate-draft` |
+| `pipelines-all` | Runs **every** published pipeline, not just the sampled one, and reports which of the nine step types actually executed |
+| `isolation` | Fans out concurrent callers, each carrying a unique marker, and asserts no response or thread ever carries another caller's |
+| `tasks` | The task board, `invoke/async` submission, and polling a task to a terminal state |
+| `metrics` | Prometheus exposition and the presence of `agentomatic_*` series |
+| `rate-limit` | That user routes *are* limited and probes and `/metrics` are *not* |
+| `auth` | Anonymous and wrong-credential rejection, valid-credential acceptance, and that every probe path stays public |
+| `errors` | Unknown agents, plugins, pipelines, endpoints and tasks return 4xx — never a 500 |
+| `control-plane` | Every read route, plus disabling an agent, confirming it stops serving, re-enabling it, and toggling maintenance |
+
+## Adapting to the deployment
+
+The harness distinguishes *not configured* from *broken*, so a lean deployment
+does not produce false failures:
+
+- **No Studio** (`--profile minimal`): pass `--no-studio`.
+- **No auth**: omit `--expect-auth`; the auth group is skipped.
+- **No control plane / no metrics / no rate limiting**: detected from the
+ response and reported as skipped.
+- **No store**: thread and optimization-run routes answer `400`; the harness
+ reports them skipped and names the variables that would enable them
+ (`DATABASE_URL`, `AGENTOMATIC_LOGS_HISTORY`).
+
+Rate limiting is handled rather than worked around: the harness is itself a
+burst of traffic from one IP, so it honours `Retry-After` and retries — except
+where a `429` is the property under test.
+
+## Durability: does the data outlive the container?
+
+Every check above runs against one live process, and a store that quietly fell
+back to a file inside the container passes all of them — it writes, it reads
+back, and only a restart tells the two apart. `durability_verify.py` splits the
+proof across a restart so the difference shows:
+
+```bash
+python scripts/durability_verify.py write \
+ --base-url http://localhost:8000 --api-key "$KEY" --agent my_chatbot
+
+# Replace the deployment: destroy the container and start a new one from the
+# same image against the same database. A restart that keeps the writable
+# layer proves nothing.
+docker rm -f my-agent && docker run -d --name my-agent … my-image
+
+python scripts/durability_verify.py verify \
+ --base-url http://localhost:8000 --api-key "$KEY" --agent my_chatbot
+```
+
+The `verify` phase reads the thread back, checks each message survived, and
+appends one more to confirm the new process can *continue* the conversation
+rather than merely read it.
+
+!!! tip "Watch the boot log for a store you did not choose"
+ Two things can silently redirect the store away from `DATABASE_URL`: a
+ MEMORY-purpose connection (which outranks it, and says so with a warning
+ naming both), and no configuration at all (which falls back to a local
+ file). Both look identical until the container is replaced.
+
+## What it does not cover
+
+- **Model quality.** Agents are exercised for wiring, not for answer quality.
+ An agent whose LLM is unreachable still passes if it degrades as designed;
+ use `agentomatic optimize` and your own eval set for quality.
+- **Your business logic.** The harness verifies the contract the platform
+ publishes. Correctness of what your nodes compute is yours to test — see
+ [Testing Your Agents](testing.md).
+- **Load and soak behaviour.** It is a correctness check, not a benchmark.
diff --git a/mkdocs.yml b/mkdocs.yml
index b8b6f83..47f5ec8 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -133,6 +133,7 @@ nav:
- Testing Your Agents: guide/testing.md
- Advanced:
- Production Deployment: guide/deployment.md
+ - Verifying a Deployment: guide/verifying-a-deployment.md
- Platform Features: guide/platform-features.md
- Custom Endpoints: guide/endpoints.md
- Tasks & Execution Modes: guide/tasks.md
diff --git a/optimization_results/.fit/marker_agent/fit_result_12905a65554d.json b/optimization_results/.fit/marker_agent/fit_result_12905a65554d.json
new file mode 100644
index 0000000..6114525
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_12905a65554d.json
@@ -0,0 +1,149 @@
+{
+ "experiment_id": "12905a65554d",
+ "agent": "marker_agent",
+ "best_score": 0.2867142857142857,
+ "baseline_score": 0.2867142857142857,
+ "absolute_improvement": 0.0,
+ "improved": false,
+ "score_history": [
+ 0.2867142857142857,
+ 0.2867142857142857
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query.",
+ "score": 0.2867142857142857,
+ "dims": {},
+ "accepted": false,
+ "what_worked": [],
+ "what_failed": [
+ "q='When is the next release candidate?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '",
+ "q='Who approved the scope change?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '",
+ "q='List the next steps for the delivery plan.' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '",
+ "q='What follow-up is needed after the demo?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '",
+ "q='What happened at the steering committee?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '"
+ ],
+ "judge_insights": [
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);"
+ ],
+ "next_focus": [
+ "Address lowest-scoring failure modes without hardcoding those exact queries."
+ ],
+ "candidate_name": "",
+ "train_score": 0.2867142857142857,
+ "holdout_score": 0.2867142857142857,
+ "generalization_gap": 0.0,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 0.2867142857142857,
+ "baseline_holdout_score": 0.2867142857142857,
+ "generalization_gap": 0.0,
+ "metric_deltas": {
+ "composite": 0.0
+ },
+ "param_suggestions": {},
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query.",
+ "user_template": null,
+ "few_shot_examples": [],
+ "output_contract": null,
+ "model_params": {},
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query.",
+ "user_template": null,
+ "few_shot_examples": [],
+ "output_contract": null,
+ "model_params": {},
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "gepa_000_0",
+ "source": "gepa_like",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "GEPA mutation 0 targeting: completeness — ensure answers cover all parts of the question. Based on 8 feedback items.",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ },
+ {
+ "round": 1,
+ "name": "gepa_000_1",
+ "source": "gepa_like",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "GEPA mutation 1 targeting: factual grounding — ensure answers are accurate and evidence-based. Based on 8 feedback items.",
+ "critique": "",
+ "resource_versions": 4,
+ "trace_count": 20
+ },
+ {
+ "round": 1,
+ "name": "gepa_000_2",
+ "source": "gepa_like",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "GEPA mutation 2 targeting: format compliance — ensure answers follow the requested structure. Based on 8 feedback items.",
+ "critique": "",
+ "resource_versions": 5,
+ "trace_count": 25
+ }
+ ],
+ "suggestions": [
+ "No configuration changes improved over the baseline."
+ ],
+ "duration_seconds": 0.42,
+ "applied": false,
+ "optimizer_name": "gepa_like",
+ "early_stop_reason": "no improvement for 1 round(s) (patience=1, monitor=best_score)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_12905a65554d",
+ "confidence": "no_improvement",
+ "deployment_recommendation": {
+ "rollout": "hold",
+ "weight": 0.0,
+ "monitoring_hours": 0
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.0,
+ "baseline_score": 0.2867,
+ "projected_score": 0.2867
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/fit_result_142c287799f8.json b/optimization_results/.fit/marker_agent/fit_result_142c287799f8.json
new file mode 100644
index 0000000..d0b330c
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_142c287799f8.json
@@ -0,0 +1,385 @@
+{
+ "experiment_id": "142c287799f8",
+ "agent": "marker_agent",
+ "best_score": 1.0009999999999997,
+ "baseline_score": 1.0009999999999997,
+ "absolute_improvement": 0.0,
+ "improved": false,
+ "score_history": [
+ 1.0009999999999997,
+ 1.0009999999999997
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query.",
+ "score": 1.0009999999999997,
+ "dims": {},
+ "accepted": false,
+ "what_worked": [
+ "q='What follow-up is needed after the demo?' score=1.00 | response contains all required markers",
+ "q='What happened at the steering committee?' score=1.00 | response contains all required markers",
+ "q='What did the customer ask for in the last meeting?' score=1.00 | response contains all required markers",
+ "q='Which teams need to be unblocked this week?' score=1.00 | response contains all required markers",
+ "q='What is the budget for Q3?' score=1.00 | response contains all required markers"
+ ],
+ "what_failed": [],
+ "judge_insights": [
+ "response contains all required markers"
+ ],
+ "next_focus": [
+ "Preserve strengths; tighten output contract and edge-case coverage."
+ ],
+ "candidate_name": "",
+ "train_score": 1.0009999999999997,
+ "holdout_score": 1.001,
+ "generalization_gap": -2.220446049250313e-16,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 1.001,
+ "baseline_holdout_score": 1.001,
+ "generalization_gap": -2.220446049250313e-16,
+ "metric_deltas": {
+ "composite": 0.0
+ },
+ "param_suggestions": {},
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "What happened at the steering committee?",
+ "response": "PARTIAL: answer to What happened at the steering committee? banana strawberry blueberry"
+ },
+ {
+ "query": "What follow-up is needed after the demo?",
+ "response": "PARTIAL: answer to What follow-up is needed after the demo? banana strawberry blueberry"
+ },
+ {
+ "query": "What is the budget for Q3?",
+ "response": "PARTIAL: answer to What is the budget for Q3? banana strawberry blueberry"
+ },
+ {
+ "query": "When is the next release candidate?",
+ "response": "PARTIAL: answer to When is the next release candidate? banana strawberry blueberry"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.15
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "What happened at the steering committee?",
+ "response": "PARTIAL: answer to What happened at the steering committee? banana strawberry blueberry"
+ },
+ {
+ "query": "What follow-up is needed after the demo?",
+ "response": "PARTIAL: answer to What follow-up is needed after the demo? banana strawberry blueberry"
+ },
+ {
+ "query": "What is the budget for Q3?",
+ "response": "PARTIAL: answer to What is the budget for Q3? banana strawberry blueberry"
+ },
+ {
+ "query": "When is the next release candidate?",
+ "response": "PARTIAL: answer to When is the next release candidate? banana strawberry blueberry"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.15
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "mipro_000_00",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 1.001,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 0: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.15}",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_01",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 1: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.65}",
+ "critique": "",
+ "resource_versions": 4,
+ "trace_count": 20
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_02",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 2: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.6}",
+ "critique": "",
+ "resource_versions": 5,
+ "trace_count": 25
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_03",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 3: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.7}",
+ "critique": "",
+ "resource_versions": 6,
+ "trace_count": 30
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_04",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 1.001,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 4: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.05}",
+ "critique": "",
+ "resource_versions": 7,
+ "trace_count": 35
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_05",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 1.001,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 5: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.15}",
+ "critique": "",
+ "resource_versions": 8,
+ "trace_count": 40
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_06",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 6: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.65}",
+ "critique": "",
+ "resource_versions": 9,
+ "trace_count": 45
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_07",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 7: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.6}",
+ "critique": "",
+ "resource_versions": 10,
+ "trace_count": 50
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_08",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 8: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.7}",
+ "critique": "",
+ "resource_versions": 11,
+ "trace_count": 55
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_09",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 1.001,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 9: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.05}",
+ "critique": "",
+ "resource_versions": 12,
+ "trace_count": 60
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_10",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 1.001,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 10: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.15}",
+ "critique": "",
+ "resource_versions": 13,
+ "trace_count": 65
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_11",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 11: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.65}",
+ "critique": "",
+ "resource_versions": 14,
+ "trace_count": 70
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_12",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 12: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.6}",
+ "critique": "",
+ "resource_versions": 15,
+ "trace_count": 75
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_13",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 13: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.7}",
+ "critique": "",
+ "resource_versions": 16,
+ "trace_count": 80
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_14",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 1.001,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 14: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.05}",
+ "critique": "",
+ "resource_versions": 17,
+ "trace_count": 85
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_00",
+ "source": "mipro_like",
+ "phase": "full_val",
+ "score": 1.0009999999999997,
+ "holdout_score": 1.001,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=1.0010, holdout=1.0010, gap=-0.0000",
+ "fit_score": 1.0009999999999997,
+ "holdout_score": 1.001,
+ "gap": -2.220446049250313e-16,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant. Always include 'Judge', 'OPT', 'banana', 'guidance' in your answer, exactly as written, for every query.",
+ "prompt_preview": "You are a helpful AI assistant. Always include 'Judge', 'OPT', 'banana', 'guidance' in your answer, exactly as written, for every query."
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_04",
+ "source": "mipro_like",
+ "phase": "full_val",
+ "score": 1.0009999999999997,
+ "holdout_score": 1.001,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=1.0010, holdout=1.0010, gap=-0.0000",
+ "fit_score": 1.0009999999999997,
+ "holdout_score": 1.001,
+ "gap": -2.220446049250313e-16,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'Judge', 'guidance' in your answer, exactly as written, for every query.",
+ "prompt_preview": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'Judge', 'guidance' in your answer, exactly as written, for every query."
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_05",
+ "source": "mipro_like",
+ "phase": "full_val",
+ "score": 1.0009999999999997,
+ "holdout_score": 1.001,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=1.0010, holdout=1.0010, gap=-0.0000",
+ "fit_score": 1.0009999999999997,
+ "holdout_score": 1.001,
+ "gap": -2.220446049250313e-16,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'Judge', 'guidance' in your answer, exactly as written, for every query.",
+ "prompt_preview": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'Judge', 'guidance' in your answer, exactly as written, for every query."
+ }
+ ],
+ "suggestions": [
+ "Baseline already saturated at 1.0010. The fit metric is too easy (or the dataset is trivial) — prompt candidates cannot show improvement. Harden the metric (content overlap / must_include / judge rubric) and expand demos.",
+ "No configuration changes improved over the baseline."
+ ],
+ "duration_seconds": 0.59,
+ "applied": false,
+ "optimizer_name": "mipro_like",
+ "early_stop_reason": "no improvement for 1 round(s) (patience=1, monitor=best_score)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_142c287799f8",
+ "confidence": "no_improvement",
+ "model_params": {
+ "temperature": 0.15
+ },
+ "deployment_recommendation": {
+ "rollout": "hold",
+ "weight": 0.0,
+ "monitoring_hours": 0
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.0,
+ "baseline_score": 1.001,
+ "projected_score": 1.001
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/fit_result_16acabe00699.json b/optimization_results/.fit/marker_agent/fit_result_16acabe00699.json
new file mode 100644
index 0000000..a24b7aa
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_16acabe00699.json
@@ -0,0 +1,385 @@
+{
+ "experiment_id": "16acabe00699",
+ "agent": "marker_agent",
+ "best_score": 1.0009999999999997,
+ "baseline_score": 1.0009999999999997,
+ "absolute_improvement": 0.0,
+ "improved": false,
+ "score_history": [
+ 1.0009999999999997,
+ 1.0009999999999997
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query.",
+ "score": 1.0009999999999997,
+ "dims": {},
+ "accepted": false,
+ "what_worked": [
+ "q='What follow-up is needed after the demo?' score=1.00 | response contains all required markers",
+ "q='What happened at the steering committee?' score=1.00 | response contains all required markers",
+ "q='What did the customer ask for in the last meeting?' score=1.00 | response contains all required markers",
+ "q='Which teams need to be unblocked this week?' score=1.00 | response contains all required markers",
+ "q='What is the budget for Q3?' score=1.00 | response contains all required markers"
+ ],
+ "what_failed": [],
+ "judge_insights": [
+ "response contains all required markers"
+ ],
+ "next_focus": [
+ "Preserve strengths; tighten output contract and edge-case coverage."
+ ],
+ "candidate_name": "",
+ "train_score": 1.0009999999999997,
+ "holdout_score": 1.001,
+ "generalization_gap": -2.220446049250313e-16,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 1.001,
+ "baseline_holdout_score": 1.001,
+ "generalization_gap": -2.220446049250313e-16,
+ "metric_deltas": {
+ "composite": 0.0
+ },
+ "param_suggestions": {},
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "What happened at the steering committee?",
+ "response": "PARTIAL: answer to What happened at the steering committee? banana strawberry blueberry"
+ },
+ {
+ "query": "What follow-up is needed after the demo?",
+ "response": "PARTIAL: answer to What follow-up is needed after the demo? banana strawberry blueberry"
+ },
+ {
+ "query": "What is the budget for Q3?",
+ "response": "PARTIAL: answer to What is the budget for Q3? banana strawberry blueberry"
+ },
+ {
+ "query": "When is the next release candidate?",
+ "response": "PARTIAL: answer to When is the next release candidate? banana strawberry blueberry"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.15
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "What happened at the steering committee?",
+ "response": "PARTIAL: answer to What happened at the steering committee? banana strawberry blueberry"
+ },
+ {
+ "query": "What follow-up is needed after the demo?",
+ "response": "PARTIAL: answer to What follow-up is needed after the demo? banana strawberry blueberry"
+ },
+ {
+ "query": "What is the budget for Q3?",
+ "response": "PARTIAL: answer to What is the budget for Q3? banana strawberry blueberry"
+ },
+ {
+ "query": "When is the next release candidate?",
+ "response": "PARTIAL: answer to When is the next release candidate? banana strawberry blueberry"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.15
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "mipro_000_00",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.7152857142857143,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 0: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.45}",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_01",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 1: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.7}",
+ "critique": "",
+ "resource_versions": 4,
+ "trace_count": 20
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_02",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 2: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.6}",
+ "critique": "",
+ "resource_versions": 5,
+ "trace_count": 25
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_03",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 3: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.55}",
+ "critique": "",
+ "resource_versions": 6,
+ "trace_count": 30
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_04",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 1.001,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 4: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.1}",
+ "critique": "",
+ "resource_versions": 7,
+ "trace_count": 35
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_05",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.7152857142857143,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 5: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.45}",
+ "critique": "",
+ "resource_versions": 8,
+ "trace_count": 40
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_06",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 6: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.7}",
+ "critique": "",
+ "resource_versions": 9,
+ "trace_count": 45
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_07",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 7: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.6}",
+ "critique": "",
+ "resource_versions": 10,
+ "trace_count": 50
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_08",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 8: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.55}",
+ "critique": "",
+ "resource_versions": 11,
+ "trace_count": 55
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_09",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 1.001,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 9: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.1}",
+ "critique": "",
+ "resource_versions": 12,
+ "trace_count": 60
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_10",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.7152857142857143,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 10: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.45}",
+ "critique": "",
+ "resource_versions": 13,
+ "trace_count": 65
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_11",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 11: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.7}",
+ "critique": "",
+ "resource_versions": 14,
+ "trace_count": 70
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_12",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 12: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.6}",
+ "critique": "",
+ "resource_versions": 15,
+ "trace_count": 75
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_13",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 13: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.55}",
+ "critique": "",
+ "resource_versions": 16,
+ "trace_count": 80
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_14",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 1.001,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 14: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.1}",
+ "critique": "",
+ "resource_versions": 17,
+ "trace_count": 85
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_04",
+ "source": "mipro_like",
+ "phase": "full_val",
+ "score": 1.0009999999999997,
+ "holdout_score": 1.001,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=1.0010, holdout=1.0010, gap=-0.0000",
+ "fit_score": 1.0009999999999997,
+ "holdout_score": 1.001,
+ "gap": -2.220446049250313e-16,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'Judge', 'guidance' in your answer, exactly as written, for every query.",
+ "prompt_preview": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'Judge', 'guidance' in your answer, exactly as written, for every query."
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_09",
+ "source": "mipro_like",
+ "phase": "full_val",
+ "score": 1.0009999999999997,
+ "holdout_score": 1.001,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=1.0010, holdout=1.0010, gap=-0.0000",
+ "fit_score": 1.0009999999999997,
+ "holdout_score": 1.001,
+ "gap": -2.220446049250313e-16,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'Judge', 'guidance' in your answer, exactly as written, for every query.",
+ "prompt_preview": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'Judge', 'guidance' in your answer, exactly as written, for every query."
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_14",
+ "source": "mipro_like",
+ "phase": "full_val",
+ "score": 1.0009999999999997,
+ "holdout_score": 1.001,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=1.0010, holdout=1.0010, gap=-0.0000",
+ "fit_score": 1.0009999999999997,
+ "holdout_score": 1.001,
+ "gap": -2.220446049250313e-16,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'Judge', 'guidance' in your answer, exactly as written, for every query.",
+ "prompt_preview": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'Judge', 'guidance' in your answer, exactly as written, for every query."
+ }
+ ],
+ "suggestions": [
+ "Baseline already saturated at 1.0010. The fit metric is too easy (or the dataset is trivial) — prompt candidates cannot show improvement. Harden the metric (content overlap / must_include / judge rubric) and expand demos.",
+ "No configuration changes improved over the baseline."
+ ],
+ "duration_seconds": 0.72,
+ "applied": false,
+ "optimizer_name": "mipro_like",
+ "early_stop_reason": "no improvement for 1 round(s) (patience=1, monitor=best_score)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_16acabe00699",
+ "confidence": "no_improvement",
+ "model_params": {
+ "temperature": 0.15
+ },
+ "deployment_recommendation": {
+ "rollout": "hold",
+ "weight": 0.0,
+ "monitoring_hours": 0
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.0,
+ "baseline_score": 1.001,
+ "projected_score": 1.001
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/fit_result_1ae10da0d0e0.json b/optimization_results/.fit/marker_agent/fit_result_1ae10da0d0e0.json
new file mode 100644
index 0000000..d1d2424
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_1ae10da0d0e0.json
@@ -0,0 +1,151 @@
+{
+ "experiment_id": "1ae10da0d0e0",
+ "agent": "marker_agent",
+ "best_score": 0.14385714285714288,
+ "baseline_score": 0.14385714285714288,
+ "absolute_improvement": 0.0,
+ "improved": false,
+ "score_history": [
+ 0.14385714285714288,
+ 0.14385714285714288
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant.\n\n## Fit tips (from labelled demos)\n- Ground every answer ONLY in the provided snapshot; never invent budgets or stakeholders.\n- Always return non-empty JSON keys `content` and `next_action`.\n- Prefer concrete next actions (≥4 words) tied to unknowns/status.\n- When relevant, include these anchors naturally: banana.",
+ "score": 0.14385714285714288,
+ "dims": {},
+ "accepted": false,
+ "what_worked": [],
+ "what_failed": [
+ "q='When is the next release candidate?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='Who approved the scope change?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='List the next steps for the delivery plan.' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='What follow-up is needed after the demo?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='What happened at the steering committee?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token "
+ ],
+ "judge_insights": [
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an"
+ ],
+ "next_focus": [
+ "Address lowest-scoring failure modes without hardcoding those exact queries."
+ ],
+ "candidate_name": "",
+ "train_score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "generalization_gap": 2.7755575615628914e-17,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 0.14385714285714285,
+ "baseline_holdout_score": 0.14385714285714285,
+ "generalization_gap": 2.7755575615628914e-17,
+ "metric_deltas": {
+ "composite": 0.0
+ },
+ "param_suggestions": {},
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant.\n\n## Fit tips (from labelled demos)\n- Ground every answer ONLY in the provided snapshot; never invent budgets or stakeholders.\n- Always return non-empty JSON keys `content` and `next_action`.\n- Prefer concrete next actions (≥4 words) tied to unknowns/status.\n- When relevant, include these anchors naturally: banana.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "Who is accountable for the migration?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ },
+ {
+ "query": "What are the open blockers for the release?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ },
+ {
+ "query": "What was decided about the API contract?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {},
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant.\n\n## Fit tips (from labelled demos)\n- Ground every answer ONLY in the provided snapshot; never invent budgets or stakeholders.\n- Always return non-empty JSON keys `content` and `next_action`.\n- Prefer concrete next actions (≥4 words) tied to unknowns/status.\n- When relevant, include these anchors naturally: banana.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "Who is accountable for the migration?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ },
+ {
+ "query": "What are the open blockers for the release?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ },
+ {
+ "query": "What was decided about the API contract?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {},
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "rewrite_000",
+ "source": "rewrite",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Full prompt rewrite at iteration 0 (3 pass(es): pass1_draft chars=452 style=slm, pass2_critique chars=326, pass3_revise chars=452). Analysed 6 failures (avg score 0.144) and 6 successes. Context: 1 rounds history, baseline=0.144, current=0.144.",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ }
+ ],
+ "suggestions": [
+ "No configuration changes improved over the baseline."
+ ],
+ "duration_seconds": 0.37,
+ "applied": false,
+ "optimizer_name": "rewrite",
+ "early_stop_reason": "no improvement for 1 round(s) (patience=1, monitor=best_score)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_1ae10da0d0e0",
+ "confidence": "no_improvement",
+ "deployment_recommendation": {
+ "rollout": "hold",
+ "weight": 0.0,
+ "monitoring_hours": 0
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.0,
+ "baseline_score": 0.1439,
+ "projected_score": 0.1439
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/fit_result_23c74d4ea257.json b/optimization_results/.fit/marker_agent/fit_result_23c74d4ea257.json
new file mode 100644
index 0000000..003d0f0
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_23c74d4ea257.json
@@ -0,0 +1,399 @@
+{
+ "experiment_id": "23c74d4ea257",
+ "agent": "marker_agent",
+ "best_score": 0.5724285714285714,
+ "baseline_score": 0.4295714285714286,
+ "absolute_improvement": 0.1428571428571428,
+ "improved": true,
+ "score_history": [
+ 0.4295714285714286,
+ 0.5724285714285714
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query.",
+ "score": 0.5724285714285714,
+ "dims": {},
+ "accepted": true,
+ "what_worked": [],
+ "what_failed": [],
+ "judge_insights": [
+ "response is missing the 'r1' marker (temperature must be <= 0.5); response is missing the 'r2' marker (temperature must be <= 0.3); response is missing the 'r3' marker (temperature must be <= 0.15)"
+ ],
+ "next_focus": [
+ "Preserve strengths; tighten output contract and edge-case coverage."
+ ],
+ "candidate_name": "mipro_000_03",
+ "train_score": 0.5724285714285714,
+ "holdout_score": 0.5724285714285714,
+ "generalization_gap": 0.0,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 0.5724285714285714,
+ "baseline_holdout_score": 0.42957142857142855,
+ "generalization_gap": 0.0,
+ "metric_deltas": {
+ "composite": 0.1429
+ },
+ "param_suggestions": {
+ "system_prompt": {
+ "param_name": "system_prompt",
+ "old_value": "[142 chars]",
+ "new_value": "[132 chars]",
+ "reason": "System prompt revised (142 → 132 chars)"
+ },
+ "few_shot_examples": {
+ "param_name": "few_shot_examples",
+ "old_value": 4,
+ "new_value": 4,
+ "reason": "Few-shot examples changed (4 → 4)"
+ },
+ "model_params.temperature": {
+ "param_name": "model_params.temperature",
+ "old_value": 0.6,
+ "new_value": 0.15,
+ "reason": "Model param 'temperature': 0.6 → 0.15"
+ }
+ },
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "What happened at the steering committee?",
+ "response": "PARTIAL: answer to What happened at the steering committee? banana strawberry blueberry"
+ },
+ {
+ "query": "What follow-up is needed after the demo?",
+ "response": "PARTIAL: answer to What follow-up is needed after the demo? banana strawberry blueberry"
+ },
+ {
+ "query": "What is the budget for Q3?",
+ "response": "PARTIAL: answer to What is the budget for Q3? banana strawberry blueberry"
+ },
+ {
+ "query": "When is the next release candidate?",
+ "response": "PARTIAL: answer to When is the next release candidate? banana strawberry blueberry"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.15
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant. Always include 'blueberry', 'strawberry', 'OPT', 'banana' in your answer, exactly as written, for every query.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "What is the budget for Q3?",
+ "response": "PARTIAL: answer to What is the budget for Q3? banana"
+ },
+ {
+ "query": "List the next steps for the delivery plan.",
+ "response": "PARTIAL: answer to List the next steps for the delivery plan. banana"
+ },
+ {
+ "query": "What did the customer ask for in the last meeting?",
+ "response": "PARTIAL: answer to What did the customer ask for in the last meeting? banana"
+ },
+ {
+ "query": "What happened at the steering committee?",
+ "response": "PARTIAL: answer to What happened at the steering committee? banana"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.6
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "mipro_000_00",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.42957142857142855,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 0: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.7}",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_01",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.42957142857142855,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 1: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.35}",
+ "critique": "",
+ "resource_versions": 4,
+ "trace_count": 20
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_02",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.42957142857142855,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 2: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.02}",
+ "critique": "",
+ "resource_versions": 5,
+ "trace_count": 25
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_03",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 3: instruction variant (len=132), 4 few-shot examples, params={'temperature': 0.15}",
+ "critique": "",
+ "resource_versions": 6,
+ "trace_count": 30
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_04",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 4: instruction variant (len=132), 4 few-shot examples, params={'temperature': 0.0}",
+ "critique": "",
+ "resource_versions": 7,
+ "trace_count": 35
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_05",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 5: instruction variant (len=132), 4 few-shot examples, params={'temperature': 0.7}",
+ "critique": "",
+ "resource_versions": 8,
+ "trace_count": 40
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_06",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 6: instruction variant (len=132), 4 few-shot examples, params={'temperature': 0.35}",
+ "critique": "",
+ "resource_versions": 9,
+ "trace_count": 45
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_07",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 7: instruction variant (len=132), 4 few-shot examples, params={'temperature': 0.02}",
+ "critique": "",
+ "resource_versions": 10,
+ "trace_count": 50
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_08",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 8: instruction variant (len=132), 4 few-shot examples, params={'temperature': 0.15}",
+ "critique": "",
+ "resource_versions": 11,
+ "trace_count": 55
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_09",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 9: instruction variant (len=132), 4 few-shot examples, params={'temperature': 0.0}",
+ "critique": "",
+ "resource_versions": 12,
+ "trace_count": 60
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_10",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 10: instruction variant (len=132), 4 few-shot examples, params={'temperature': 0.7}",
+ "critique": "",
+ "resource_versions": 13,
+ "trace_count": 65
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_11",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 11: instruction variant (len=132), 4 few-shot examples, params={'temperature': 0.35}",
+ "critique": "",
+ "resource_versions": 14,
+ "trace_count": 70
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_12",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 12: instruction variant (len=132), 4 few-shot examples, params={'temperature': 0.02}",
+ "critique": "",
+ "resource_versions": 15,
+ "trace_count": 75
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_13",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 13: instruction variant (len=132), 4 few-shot examples, params={'temperature': 0.15}",
+ "critique": "",
+ "resource_versions": 16,
+ "trace_count": 80
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_14",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 14: instruction variant (len=132), 4 few-shot examples, params={'temperature': 0.0}",
+ "critique": "",
+ "resource_versions": 17,
+ "trace_count": 85
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_03",
+ "source": "mipro_like",
+ "phase": "full_val",
+ "score": 0.5724285714285714,
+ "holdout_score": 0.5724285714285714,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=0.5724, holdout=0.5724, gap=+0.0000",
+ "fit_score": 0.5724285714285714,
+ "holdout_score": 0.5724285714285714,
+ "gap": 0.0,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query.",
+ "prompt_preview": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query."
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_04",
+ "source": "mipro_like",
+ "phase": "full_val",
+ "score": 0.5724285714285714,
+ "holdout_score": 0.5724285714285714,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=0.5724, holdout=0.5724, gap=+0.0000",
+ "fit_score": 0.5724285714285714,
+ "holdout_score": 0.5724285714285714,
+ "gap": 0.0,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query.",
+ "prompt_preview": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query."
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_05",
+ "source": "mipro_like",
+ "phase": "full_val",
+ "score": 0.5724285714285714,
+ "holdout_score": 0.5724285714285714,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=0.5724, holdout=0.5724, gap=+0.0000",
+ "fit_score": 0.5724285714285714,
+ "holdout_score": 0.5724285714285714,
+ "gap": 0.0,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query.",
+ "prompt_preview": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query."
+ }
+ ],
+ "suggestions": [
+ "System prompt revised (142 → 132 chars)",
+ "Few-shot examples changed (4 → 4)",
+ "Model param 'temperature': 0.6 → 0.15"
+ ],
+ "duration_seconds": 0.66,
+ "applied": false,
+ "optimizer_name": "mipro_like",
+ "early_stop_reason": "completed all 1 optimize round(s) (max_trials=6)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_23c74d4ea257",
+ "confidence": "high",
+ "model_params": {
+ "temperature": 0.15
+ },
+ "deployment_recommendation": {
+ "rollout": "canary",
+ "weight": 0.4,
+ "monitoring_hours": 12
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.1429,
+ "baseline_score": 0.4296,
+ "projected_score": 0.5724
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/fit_result_28f28abcee98.json b/optimization_results/.fit/marker_agent/fit_result_28f28abcee98.json
new file mode 100644
index 0000000..2e042d7
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_28f28abcee98.json
@@ -0,0 +1,151 @@
+{
+ "experiment_id": "28f28abcee98",
+ "agent": "marker_agent",
+ "best_score": 0.14385714285714288,
+ "baseline_score": 0.14385714285714288,
+ "absolute_improvement": 0.0,
+ "improved": false,
+ "score_history": [
+ 0.14385714285714288,
+ 0.14385714285714288
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant.\n\n## Fit tips (from labelled demos)\n- Ground every answer ONLY in the provided snapshot; never invent budgets or stakeholders.\n- Always return non-empty JSON keys `content` and `next_action`.\n- Prefer concrete next actions (≥4 words) tied to unknowns/status.\n- When relevant, include these anchors naturally: banana.",
+ "score": 0.14385714285714288,
+ "dims": {},
+ "accepted": false,
+ "what_worked": [],
+ "what_failed": [
+ "q='When is the next release candidate?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='Who approved the scope change?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='List the next steps for the delivery plan.' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='What follow-up is needed after the demo?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='What happened at the steering committee?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token "
+ ],
+ "judge_insights": [
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an"
+ ],
+ "next_focus": [
+ "Address lowest-scoring failure modes without hardcoding those exact queries."
+ ],
+ "candidate_name": "",
+ "train_score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "generalization_gap": 2.7755575615628914e-17,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 0.14385714285714285,
+ "baseline_holdout_score": 0.14385714285714285,
+ "generalization_gap": 2.7755575615628914e-17,
+ "metric_deltas": {
+ "composite": 0.0
+ },
+ "param_suggestions": {},
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant.\n\n## Fit tips (from labelled demos)\n- Ground every answer ONLY in the provided snapshot; never invent budgets or stakeholders.\n- Always return non-empty JSON keys `content` and `next_action`.\n- Prefer concrete next actions (≥4 words) tied to unknowns/status.\n- When relevant, include these anchors naturally: banana.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "Who is accountable for the migration?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ },
+ {
+ "query": "What are the open blockers for the release?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ },
+ {
+ "query": "What was decided about the API contract?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {},
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant.\n\n## Fit tips (from labelled demos)\n- Ground every answer ONLY in the provided snapshot; never invent budgets or stakeholders.\n- Always return non-empty JSON keys `content` and `next_action`.\n- Prefer concrete next actions (≥4 words) tied to unknowns/status.\n- When relevant, include these anchors naturally: banana.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "Who is accountable for the migration?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ },
+ {
+ "query": "What are the open blockers for the release?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ },
+ {
+ "query": "What was decided about the API contract?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {},
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "rewrite_000",
+ "source": "rewrite",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Full prompt rewrite at iteration 0 (3 pass(es): pass1_draft chars=452 style=slm, pass2_critique chars=326, pass3_revise chars=452). Analysed 6 failures (avg score 0.144) and 6 successes. Context: 1 rounds history, baseline=0.144, current=0.144.",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ }
+ ],
+ "suggestions": [
+ "No configuration changes improved over the baseline."
+ ],
+ "duration_seconds": 0.35,
+ "applied": false,
+ "optimizer_name": "rewrite",
+ "early_stop_reason": "no improvement for 1 round(s) (patience=1, monitor=best_score)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_28f28abcee98",
+ "confidence": "no_improvement",
+ "deployment_recommendation": {
+ "rollout": "hold",
+ "weight": 0.0,
+ "monitoring_hours": 0
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.0,
+ "baseline_score": 0.1439,
+ "projected_score": 0.1439
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/fit_result_296d3b2b6cf1.json b/optimization_results/.fit/marker_agent/fit_result_296d3b2b6cf1.json
new file mode 100644
index 0000000..0af8950
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_296d3b2b6cf1.json
@@ -0,0 +1,412 @@
+{
+ "experiment_id": "296d3b2b6cf1",
+ "agent": "marker_agent",
+ "best_score": 0.2867142857142857,
+ "baseline_score": 0.14385714285714288,
+ "absolute_improvement": 0.14285714285714282,
+ "improved": true,
+ "score_history": [
+ 0.14385714285714288,
+ 0.2867142857142857
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant. Always include 'blueberry', 'strawberry', 'OPT', 'banana' in your answer, exactly as written, for every query.",
+ "score": 0.2867142857142857,
+ "dims": {},
+ "accepted": true,
+ "what_worked": [],
+ "what_failed": [
+ "q='When is the next release candidate?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '",
+ "q='Who approved the scope change?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '",
+ "q='List the next steps for the delivery plan.' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '",
+ "q='What follow-up is needed after the demo?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '",
+ "q='What happened at the steering committee?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '"
+ ],
+ "judge_insights": [
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);"
+ ],
+ "next_focus": [
+ "Address lowest-scoring failure modes without hardcoding those exact queries."
+ ],
+ "candidate_name": "mipro_000_03",
+ "train_score": 0.2867142857142857,
+ "holdout_score": 0.2867142857142857,
+ "generalization_gap": 0.0,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 0.2867142857142857,
+ "baseline_holdout_score": 0.14385714285714285,
+ "generalization_gap": 0.0,
+ "metric_deltas": {
+ "composite": 0.1429
+ },
+ "param_suggestions": {
+ "system_prompt": {
+ "param_name": "system_prompt",
+ "old_value": "[136 chars]",
+ "new_value": "[142 chars]",
+ "reason": "System prompt revised (136 → 142 chars)"
+ },
+ "few_shot_examples": {
+ "param_name": "few_shot_examples",
+ "old_value": 4,
+ "new_value": 4,
+ "reason": "Few-shot examples changed (4 → 4)"
+ },
+ "model_params.temperature": {
+ "param_name": "model_params.temperature",
+ "old_value": 0.5,
+ "new_value": 0.6,
+ "reason": "Model param 'temperature': 0.5 → 0.6"
+ }
+ },
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant. Always include 'blueberry', 'strawberry', 'OPT', 'banana' in your answer, exactly as written, for every query.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "What is the budget for Q3?",
+ "response": "PARTIAL: answer to What is the budget for Q3? banana"
+ },
+ {
+ "query": "List the next steps for the delivery plan.",
+ "response": "PARTIAL: answer to List the next steps for the delivery plan. banana"
+ },
+ {
+ "query": "What did the customer ask for in the last meeting?",
+ "response": "PARTIAL: answer to What did the customer ask for in the last meeting? banana"
+ },
+ {
+ "query": "What happened at the steering committee?",
+ "response": "PARTIAL: answer to What happened at the steering committee? banana"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.6
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant. Always include 'banana', 'Judge', 'OPT', 'guidance' in your answer, exactly as written, for every query.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "What did the customer ask for in the last meeting?",
+ "response": "BASE: answer to What did the customer ask for in the last meeting?"
+ },
+ {
+ "query": "What happened at the steering committee?",
+ "response": "BASE: answer to What happened at the steering committee?"
+ },
+ {
+ "query": "What follow-up is needed after the demo?",
+ "response": "BASE: answer to What follow-up is needed after the demo?"
+ },
+ {
+ "query": "Who approved the scope change?",
+ "response": "BASE: answer to Who approved the scope change?"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.5
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "mipro_000_00",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 0: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.35}",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_01",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 1: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.7}",
+ "critique": "",
+ "resource_versions": 4,
+ "trace_count": 20
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_02",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 2: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.65}",
+ "critique": "",
+ "resource_versions": 5,
+ "trace_count": 25
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_03",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 3: instruction variant (len=142), 4 few-shot examples, params={'temperature': 0.6}",
+ "critique": "",
+ "resource_versions": 6,
+ "trace_count": 30
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_04",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 4: instruction variant (len=142), 4 few-shot examples, params={'temperature': 0.05}",
+ "critique": "",
+ "resource_versions": 7,
+ "trace_count": 35
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_05",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 5: instruction variant (len=142), 4 few-shot examples, params={'temperature': 0.35}",
+ "critique": "",
+ "resource_versions": 8,
+ "trace_count": 40
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_06",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 6: instruction variant (len=142), 4 few-shot examples, params={'temperature': 0.7}",
+ "critique": "",
+ "resource_versions": 9,
+ "trace_count": 45
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_07",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 7: instruction variant (len=142), 4 few-shot examples, params={'temperature': 0.65}",
+ "critique": "",
+ "resource_versions": 10,
+ "trace_count": 50
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_08",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 8: instruction variant (len=142), 4 few-shot examples, params={'temperature': 0.6}",
+ "critique": "",
+ "resource_versions": 11,
+ "trace_count": 55
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_09",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 9: instruction variant (len=142), 4 few-shot examples, params={'temperature': 0.05}",
+ "critique": "",
+ "resource_versions": 12,
+ "trace_count": 60
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_10",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 10: instruction variant (len=142), 4 few-shot examples, params={'temperature': 0.35}",
+ "critique": "",
+ "resource_versions": 13,
+ "trace_count": 65
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_11",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 11: instruction variant (len=142), 4 few-shot examples, params={'temperature': 0.7}",
+ "critique": "",
+ "resource_versions": 14,
+ "trace_count": 70
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_12",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 12: instruction variant (len=142), 4 few-shot examples, params={'temperature': 0.65}",
+ "critique": "",
+ "resource_versions": 15,
+ "trace_count": 75
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_13",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 13: instruction variant (len=142), 4 few-shot examples, params={'temperature': 0.6}",
+ "critique": "",
+ "resource_versions": 16,
+ "trace_count": 80
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_14",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 14: instruction variant (len=142), 4 few-shot examples, params={'temperature': 0.05}",
+ "critique": "",
+ "resource_versions": 17,
+ "trace_count": 85
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_03",
+ "source": "mipro_like",
+ "phase": "full_val",
+ "score": 0.2867142857142857,
+ "holdout_score": 0.2867142857142857,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=0.2867, holdout=0.2867, gap=+0.0000",
+ "fit_score": 0.2867142857142857,
+ "holdout_score": 0.2867142857142857,
+ "gap": 0.0,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant. Always include 'blueberry', 'strawberry', 'OPT', 'banana' in your answer, exactly as written, for every query.",
+ "prompt_preview": "You are a helpful AI assistant. Always include 'blueberry', 'strawberry', 'OPT', 'banana' in your answer, exactly as written, for every query."
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_04",
+ "source": "mipro_like",
+ "phase": "full_val",
+ "score": 0.2867142857142857,
+ "holdout_score": 0.2867142857142857,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=0.2867, holdout=0.2867, gap=+0.0000",
+ "fit_score": 0.2867142857142857,
+ "holdout_score": 0.2867142857142857,
+ "gap": 0.0,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant. Always include 'blueberry', 'strawberry', 'OPT', 'banana' in your answer, exactly as written, for every query.",
+ "prompt_preview": "You are a helpful AI assistant. Always include 'blueberry', 'strawberry', 'OPT', 'banana' in your answer, exactly as written, for every query."
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_05",
+ "source": "mipro_like",
+ "phase": "full_val",
+ "score": 0.2867142857142857,
+ "holdout_score": 0.2867142857142857,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=0.2867, holdout=0.2867, gap=+0.0000",
+ "fit_score": 0.2867142857142857,
+ "holdout_score": 0.2867142857142857,
+ "gap": 0.0,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant. Always include 'blueberry', 'strawberry', 'OPT', 'banana' in your answer, exactly as written, for every query.",
+ "prompt_preview": "You are a helpful AI assistant. Always include 'blueberry', 'strawberry', 'OPT', 'banana' in your answer, exactly as written, for every query."
+ }
+ ],
+ "suggestions": [
+ "System prompt revised (136 → 142 chars)",
+ "Few-shot examples changed (4 → 4)",
+ "Model param 'temperature': 0.5 → 0.6"
+ ],
+ "duration_seconds": 0.62,
+ "applied": false,
+ "optimizer_name": "mipro_like",
+ "early_stop_reason": "completed all 1 optimize round(s) (max_trials=6)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_296d3b2b6cf1",
+ "confidence": "high",
+ "model_params": {
+ "temperature": 0.6
+ },
+ "deployment_recommendation": {
+ "rollout": "canary",
+ "weight": 0.4,
+ "monitoring_hours": 12
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.1429,
+ "baseline_score": 0.1439,
+ "projected_score": 0.2867
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/fit_result_2be78907a291.json b/optimization_results/.fit/marker_agent/fit_result_2be78907a291.json
new file mode 100644
index 0000000..e7fbd49
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_2be78907a291.json
@@ -0,0 +1,151 @@
+{
+ "experiment_id": "2be78907a291",
+ "agent": "marker_agent",
+ "best_score": 0.14385714285714288,
+ "baseline_score": 0.14385714285714288,
+ "absolute_improvement": 0.0,
+ "improved": false,
+ "score_history": [
+ 0.14385714285714288,
+ 0.14385714285714288
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant.\n\n## Fit tips (from labelled demos)\n- Ground every answer ONLY in the provided snapshot; never invent budgets or stakeholders.\n- Always return non-empty JSON keys `content` and `next_action`.\n- Prefer concrete next actions (≥4 words) tied to unknowns/status.\n- When relevant, include these anchors naturally: banana.",
+ "score": 0.14385714285714288,
+ "dims": {},
+ "accepted": false,
+ "what_worked": [],
+ "what_failed": [
+ "q='When is the next release candidate?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='Who approved the scope change?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='List the next steps for the delivery plan.' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='What follow-up is needed after the demo?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='What happened at the steering committee?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token "
+ ],
+ "judge_insights": [
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an"
+ ],
+ "next_focus": [
+ "Address lowest-scoring failure modes without hardcoding those exact queries."
+ ],
+ "candidate_name": "",
+ "train_score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "generalization_gap": 2.7755575615628914e-17,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 0.14385714285714285,
+ "baseline_holdout_score": 0.14385714285714285,
+ "generalization_gap": 2.7755575615628914e-17,
+ "metric_deltas": {
+ "composite": 0.0
+ },
+ "param_suggestions": {},
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant.\n\n## Fit tips (from labelled demos)\n- Ground every answer ONLY in the provided snapshot; never invent budgets or stakeholders.\n- Always return non-empty JSON keys `content` and `next_action`.\n- Prefer concrete next actions (≥4 words) tied to unknowns/status.\n- When relevant, include these anchors naturally: banana.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "Who is accountable for the migration?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ },
+ {
+ "query": "What are the open blockers for the release?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ },
+ {
+ "query": "What was decided about the API contract?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {},
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant.\n\n## Fit tips (from labelled demos)\n- Ground every answer ONLY in the provided snapshot; never invent budgets or stakeholders.\n- Always return non-empty JSON keys `content` and `next_action`.\n- Prefer concrete next actions (≥4 words) tied to unknowns/status.\n- When relevant, include these anchors naturally: banana.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "Who is accountable for the migration?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ },
+ {
+ "query": "What are the open blockers for the release?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ },
+ {
+ "query": "What was decided about the API contract?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {},
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "rewrite_000",
+ "source": "rewrite",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Full prompt rewrite at iteration 0 (3 pass(es): pass1_draft chars=452 style=slm, pass2_critique chars=326, pass3_revise chars=452). Analysed 6 failures (avg score 0.144) and 6 successes. Context: 1 rounds history, baseline=0.144, current=0.144.",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ }
+ ],
+ "suggestions": [
+ "No configuration changes improved over the baseline."
+ ],
+ "duration_seconds": 0.5,
+ "applied": false,
+ "optimizer_name": "rewrite",
+ "early_stop_reason": "no improvement for 1 round(s) (patience=1, monitor=best_score)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_2be78907a291",
+ "confidence": "no_improvement",
+ "deployment_recommendation": {
+ "rollout": "hold",
+ "weight": 0.0,
+ "monitoring_hours": 0
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.0,
+ "baseline_score": 0.1439,
+ "projected_score": 0.1439
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/fit_result_3d1f73e32b77.json b/optimization_results/.fit/marker_agent/fit_result_3d1f73e32b77.json
new file mode 100644
index 0000000..1571f03
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_3d1f73e32b77.json
@@ -0,0 +1,183 @@
+{
+ "experiment_id": "3d1f73e32b77",
+ "agent": "marker_agent",
+ "best_score": 0.14385714285714288,
+ "baseline_score": 0.14385714285714288,
+ "absolute_improvement": 0.0,
+ "improved": false,
+ "score_history": [
+ 0.14385714285714288,
+ 0.14385714285714288
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant.",
+ "score": 0.14385714285714288,
+ "dims": {},
+ "accepted": false,
+ "what_worked": [],
+ "what_failed": [
+ "q='When is the next release candidate?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='Who approved the scope change?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='List the next steps for the delivery plan.' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='What follow-up is needed after the demo?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='What happened at the steering committee?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token "
+ ],
+ "judge_insights": [
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an"
+ ],
+ "next_focus": [
+ "Address lowest-scoring failure modes without hardcoding those exact queries."
+ ],
+ "candidate_name": "",
+ "train_score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "generalization_gap": 2.7755575615628914e-17,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 0.14385714285714285,
+ "baseline_holdout_score": 0.14385714285714285,
+ "generalization_gap": 2.7755575615628914e-17,
+ "metric_deltas": {
+ "composite": 0.0
+ },
+ "param_suggestions": {},
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "What is the budget for Q3?",
+ "response": "BASE: answer to What is the budget for Q3?"
+ },
+ {
+ "query": "What did the customer ask for in the last meeting?",
+ "response": "BASE: answer to What did the customer ask for in the last meeting?"
+ },
+ {
+ "query": "Who approved the scope change?",
+ "response": "BASE: answer to Who approved the scope change?"
+ },
+ {
+ "query": "When is the next release candidate?",
+ "response": "BASE: answer to When is the next release candidate?"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {},
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "What is the budget for Q3?",
+ "response": "BASE: answer to What is the budget for Q3?"
+ },
+ {
+ "query": "What did the customer ask for in the last meeting?",
+ "response": "BASE: answer to What did the customer ask for in the last meeting?"
+ },
+ {
+ "query": "Who approved the scope change?",
+ "response": "BASE: answer to Who approved the scope change?"
+ },
+ {
+ "query": "When is the next release candidate?",
+ "response": "BASE: answer to When is the next release candidate?"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {},
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "fewshot_000_0",
+ "source": "few_shot_bootstrap",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Few-shot bootstrap (rank 0): 4 examples, avg_score=0.144, diversity=1.000, combined=0.401",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ },
+ {
+ "round": 1,
+ "name": "fewshot_000_1",
+ "source": "few_shot_bootstrap",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Few-shot bootstrap (rank 1): 4 examples, avg_score=0.144, diversity=1.000, combined=0.401",
+ "critique": "",
+ "resource_versions": 4,
+ "trace_count": 20
+ },
+ {
+ "round": 1,
+ "name": "fewshot_000_2",
+ "source": "few_shot_bootstrap",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Few-shot bootstrap (rank 2): 4 examples, avg_score=0.144, diversity=1.000, combined=0.401",
+ "critique": "",
+ "resource_versions": 5,
+ "trace_count": 25
+ }
+ ],
+ "suggestions": [
+ "No configuration changes improved over the baseline."
+ ],
+ "duration_seconds": 0.17,
+ "applied": false,
+ "optimizer_name": "few_shot_bootstrap",
+ "early_stop_reason": "no improvement for 1 round(s) (patience=1, monitor=best_score)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_3d1f73e32b77",
+ "confidence": "no_improvement",
+ "deployment_recommendation": {
+ "rollout": "hold",
+ "weight": 0.0,
+ "monitoring_hours": 0
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.0,
+ "baseline_score": 0.1439,
+ "projected_score": 0.1439
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/fit_result_46c20a9af339.json b/optimization_results/.fit/marker_agent/fit_result_46c20a9af339.json
new file mode 100644
index 0000000..4906b2d
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_46c20a9af339.json
@@ -0,0 +1,183 @@
+{
+ "experiment_id": "46c20a9af339",
+ "agent": "marker_agent",
+ "best_score": 0.14385714285714288,
+ "baseline_score": 0.14385714285714288,
+ "absolute_improvement": 0.0,
+ "improved": false,
+ "score_history": [
+ 0.14385714285714288,
+ 0.14385714285714288
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant.",
+ "score": 0.14385714285714288,
+ "dims": {},
+ "accepted": false,
+ "what_worked": [],
+ "what_failed": [
+ "q='When is the next release candidate?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='Who approved the scope change?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='List the next steps for the delivery plan.' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='What follow-up is needed after the demo?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='What happened at the steering committee?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token "
+ ],
+ "judge_insights": [
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an"
+ ],
+ "next_focus": [
+ "Address lowest-scoring failure modes without hardcoding those exact queries."
+ ],
+ "candidate_name": "",
+ "train_score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "generalization_gap": 2.7755575615628914e-17,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 0.14385714285714285,
+ "baseline_holdout_score": 0.14385714285714285,
+ "generalization_gap": 2.7755575615628914e-17,
+ "metric_deltas": {
+ "composite": 0.0
+ },
+ "param_suggestions": {},
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "What is the budget for Q3?",
+ "response": "BASE: answer to What is the budget for Q3?"
+ },
+ {
+ "query": "What did the customer ask for in the last meeting?",
+ "response": "BASE: answer to What did the customer ask for in the last meeting?"
+ },
+ {
+ "query": "Who approved the scope change?",
+ "response": "BASE: answer to Who approved the scope change?"
+ },
+ {
+ "query": "When is the next release candidate?",
+ "response": "BASE: answer to When is the next release candidate?"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {},
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "What is the budget for Q3?",
+ "response": "BASE: answer to What is the budget for Q3?"
+ },
+ {
+ "query": "What did the customer ask for in the last meeting?",
+ "response": "BASE: answer to What did the customer ask for in the last meeting?"
+ },
+ {
+ "query": "Who approved the scope change?",
+ "response": "BASE: answer to Who approved the scope change?"
+ },
+ {
+ "query": "When is the next release candidate?",
+ "response": "BASE: answer to When is the next release candidate?"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {},
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "fewshot_000_0",
+ "source": "few_shot_bootstrap",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Few-shot bootstrap (rank 0): 4 examples, avg_score=0.144, diversity=1.000, combined=0.401",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ },
+ {
+ "round": 1,
+ "name": "fewshot_000_1",
+ "source": "few_shot_bootstrap",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Few-shot bootstrap (rank 1): 4 examples, avg_score=0.144, diversity=1.000, combined=0.401",
+ "critique": "",
+ "resource_versions": 4,
+ "trace_count": 20
+ },
+ {
+ "round": 1,
+ "name": "fewshot_000_2",
+ "source": "few_shot_bootstrap",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Few-shot bootstrap (rank 2): 4 examples, avg_score=0.144, diversity=1.000, combined=0.401",
+ "critique": "",
+ "resource_versions": 5,
+ "trace_count": 25
+ }
+ ],
+ "suggestions": [
+ "No configuration changes improved over the baseline."
+ ],
+ "duration_seconds": 0.12,
+ "applied": false,
+ "optimizer_name": "few_shot_bootstrap",
+ "early_stop_reason": "no improvement for 1 round(s) (patience=1, monitor=best_score)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_46c20a9af339",
+ "confidence": "no_improvement",
+ "deployment_recommendation": {
+ "rollout": "hold",
+ "weight": 0.0,
+ "monitoring_hours": 0
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.0,
+ "baseline_score": 0.1439,
+ "projected_score": 0.1439
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/fit_result_538677bc709e.json b/optimization_results/.fit/marker_agent/fit_result_538677bc709e.json
new file mode 100644
index 0000000..2ecded8
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_538677bc709e.json
@@ -0,0 +1,228 @@
+{
+ "experiment_id": "538677bc709e",
+ "agent": "marker_agent",
+ "best_score": 0.4295714285714286,
+ "baseline_score": 0.4295714285714286,
+ "absolute_improvement": 0.0,
+ "improved": false,
+ "score_history": [
+ 0.4295714285714286,
+ 0.4295714285714286
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant.",
+ "score": 0.4295714285714286,
+ "dims": {},
+ "accepted": false,
+ "what_worked": [],
+ "what_failed": [
+ "q='When is the next release candidate?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud",
+ "q='Who approved the scope change?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud",
+ "q='List the next steps for the delivery plan.' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud",
+ "q='What follow-up is needed after the demo?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud",
+ "q='What happened at the steering committee?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud"
+ ],
+ "judge_insights": [
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response "
+ ],
+ "next_focus": [
+ "Address lowest-scoring failure modes without hardcoding those exact queries."
+ ],
+ "candidate_name": "",
+ "train_score": 0.4295714285714286,
+ "holdout_score": 0.42957142857142855,
+ "generalization_gap": 5.551115123125783e-17,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 0.42957142857142855,
+ "baseline_holdout_score": 0.42957142857142855,
+ "generalization_gap": 5.551115123125783e-17,
+ "metric_deltas": {
+ "composite": 0.0
+ },
+ "param_suggestions": {},
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant.",
+ "user_template": null,
+ "few_shot_examples": [],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.05
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant.",
+ "user_template": null,
+ "few_shot_examples": [],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.05
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "param_000_m00",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.42957142857142855,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.0",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ },
+ {
+ "round": 1,
+ "name": "param_000_m01",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.3",
+ "critique": "",
+ "resource_versions": 4,
+ "trace_count": 20
+ },
+ {
+ "round": 1,
+ "name": "param_000_m02",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.25",
+ "critique": "",
+ "resource_versions": 5,
+ "trace_count": 25
+ },
+ {
+ "round": 1,
+ "name": "param_000_m03",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.35",
+ "critique": "",
+ "resource_versions": 6,
+ "trace_count": 30
+ },
+ {
+ "round": 1,
+ "name": "param_000_m04",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.5",
+ "critique": "",
+ "resource_versions": 7,
+ "trace_count": 35
+ },
+ {
+ "round": 1,
+ "name": "param_000_m05",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.2",
+ "critique": "",
+ "resource_versions": 8,
+ "trace_count": 40
+ },
+ {
+ "round": 1,
+ "name": "param_000_m06",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.45",
+ "critique": "",
+ "resource_versions": 9,
+ "trace_count": 45
+ },
+ {
+ "round": 1,
+ "name": "param_000_m07",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.001,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.7",
+ "critique": "",
+ "resource_versions": 10,
+ "trace_count": 50
+ },
+ {
+ "round": 1,
+ "name": "param_000_m08",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.42957142857142855,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.1",
+ "critique": "",
+ "resource_versions": 11,
+ "trace_count": 55
+ }
+ ],
+ "suggestions": [
+ "No configuration changes improved over the baseline."
+ ],
+ "duration_seconds": 0.13,
+ "applied": false,
+ "optimizer_name": "param_search",
+ "early_stop_reason": "no improvement for 1 round(s) (patience=1, monitor=best_score)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_538677bc709e",
+ "confidence": "no_improvement",
+ "model_params": {
+ "temperature": 0.05
+ },
+ "deployment_recommendation": {
+ "rollout": "hold",
+ "weight": 0.0,
+ "monitoring_hours": 0
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.0,
+ "baseline_score": 0.4296,
+ "projected_score": 0.4296
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/fit_result_58f5a7c0ef7c.json b/optimization_results/.fit/marker_agent/fit_result_58f5a7c0ef7c.json
new file mode 100644
index 0000000..276b128
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_58f5a7c0ef7c.json
@@ -0,0 +1,183 @@
+{
+ "experiment_id": "58f5a7c0ef7c",
+ "agent": "marker_agent",
+ "best_score": 0.14385714285714288,
+ "baseline_score": 0.14385714285714288,
+ "absolute_improvement": 0.0,
+ "improved": false,
+ "score_history": [
+ 0.14385714285714288,
+ 0.14385714285714288
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant.",
+ "score": 0.14385714285714288,
+ "dims": {},
+ "accepted": false,
+ "what_worked": [],
+ "what_failed": [
+ "q='When is the next release candidate?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='Who approved the scope change?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='List the next steps for the delivery plan.' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='What follow-up is needed after the demo?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='What happened at the steering committee?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token "
+ ],
+ "judge_insights": [
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an"
+ ],
+ "next_focus": [
+ "Address lowest-scoring failure modes without hardcoding those exact queries."
+ ],
+ "candidate_name": "",
+ "train_score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "generalization_gap": 2.7755575615628914e-17,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 0.14385714285714285,
+ "baseline_holdout_score": 0.14385714285714285,
+ "generalization_gap": 2.7755575615628914e-17,
+ "metric_deltas": {
+ "composite": 0.0
+ },
+ "param_suggestions": {},
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "What is the budget for Q3?",
+ "response": "BASE: answer to What is the budget for Q3?"
+ },
+ {
+ "query": "What did the customer ask for in the last meeting?",
+ "response": "BASE: answer to What did the customer ask for in the last meeting?"
+ },
+ {
+ "query": "Who approved the scope change?",
+ "response": "BASE: answer to Who approved the scope change?"
+ },
+ {
+ "query": "When is the next release candidate?",
+ "response": "BASE: answer to When is the next release candidate?"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {},
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "What is the budget for Q3?",
+ "response": "BASE: answer to What is the budget for Q3?"
+ },
+ {
+ "query": "What did the customer ask for in the last meeting?",
+ "response": "BASE: answer to What did the customer ask for in the last meeting?"
+ },
+ {
+ "query": "Who approved the scope change?",
+ "response": "BASE: answer to Who approved the scope change?"
+ },
+ {
+ "query": "When is the next release candidate?",
+ "response": "BASE: answer to When is the next release candidate?"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {},
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "fewshot_000_0",
+ "source": "few_shot_bootstrap",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Few-shot bootstrap (rank 0): 4 examples, avg_score=0.144, diversity=1.000, combined=0.401",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ },
+ {
+ "round": 1,
+ "name": "fewshot_000_1",
+ "source": "few_shot_bootstrap",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Few-shot bootstrap (rank 1): 4 examples, avg_score=0.144, diversity=1.000, combined=0.401",
+ "critique": "",
+ "resource_versions": 4,
+ "trace_count": 20
+ },
+ {
+ "round": 1,
+ "name": "fewshot_000_2",
+ "source": "few_shot_bootstrap",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Few-shot bootstrap (rank 2): 4 examples, avg_score=0.144, diversity=1.000, combined=0.401",
+ "critique": "",
+ "resource_versions": 5,
+ "trace_count": 25
+ }
+ ],
+ "suggestions": [
+ "No configuration changes improved over the baseline."
+ ],
+ "duration_seconds": 0.11,
+ "applied": false,
+ "optimizer_name": "few_shot_bootstrap",
+ "early_stop_reason": "no improvement for 1 round(s) (patience=1, monitor=best_score)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_58f5a7c0ef7c",
+ "confidence": "no_improvement",
+ "deployment_recommendation": {
+ "rollout": "hold",
+ "weight": 0.0,
+ "monitoring_hours": 0
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.0,
+ "baseline_score": 0.1439,
+ "projected_score": 0.1439
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/fit_result_5a5f2b0a2324.json b/optimization_results/.fit/marker_agent/fit_result_5a5f2b0a2324.json
new file mode 100644
index 0000000..9d5a983
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_5a5f2b0a2324.json
@@ -0,0 +1,149 @@
+{
+ "experiment_id": "5a5f2b0a2324",
+ "agent": "marker_agent",
+ "best_score": 0.2867142857142857,
+ "baseline_score": 0.2867142857142857,
+ "absolute_improvement": 0.0,
+ "improved": false,
+ "score_history": [
+ 0.2867142857142857,
+ 0.2867142857142857
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query.",
+ "score": 0.2867142857142857,
+ "dims": {},
+ "accepted": false,
+ "what_worked": [],
+ "what_failed": [
+ "q='When is the next release candidate?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '",
+ "q='Who approved the scope change?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '",
+ "q='List the next steps for the delivery plan.' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '",
+ "q='What follow-up is needed after the demo?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '",
+ "q='What happened at the steering committee?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '"
+ ],
+ "judge_insights": [
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);"
+ ],
+ "next_focus": [
+ "Address lowest-scoring failure modes without hardcoding those exact queries."
+ ],
+ "candidate_name": "",
+ "train_score": 0.2867142857142857,
+ "holdout_score": 0.2867142857142857,
+ "generalization_gap": 0.0,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 0.2867142857142857,
+ "baseline_holdout_score": 0.2867142857142857,
+ "generalization_gap": 0.0,
+ "metric_deltas": {
+ "composite": 0.0
+ },
+ "param_suggestions": {},
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query.",
+ "user_template": null,
+ "few_shot_examples": [],
+ "output_contract": null,
+ "model_params": {},
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query.",
+ "user_template": null,
+ "few_shot_examples": [],
+ "output_contract": null,
+ "model_params": {},
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "gepa_000_0",
+ "source": "gepa_like",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "GEPA mutation 0 targeting: completeness — ensure answers cover all parts of the question. Based on 8 feedback items.",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ },
+ {
+ "round": 1,
+ "name": "gepa_000_1",
+ "source": "gepa_like",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "GEPA mutation 1 targeting: factual grounding — ensure answers are accurate and evidence-based. Based on 8 feedback items.",
+ "critique": "",
+ "resource_versions": 4,
+ "trace_count": 20
+ },
+ {
+ "round": 1,
+ "name": "gepa_000_2",
+ "source": "gepa_like",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "GEPA mutation 2 targeting: format compliance — ensure answers follow the requested structure. Based on 8 feedback items.",
+ "critique": "",
+ "resource_versions": 5,
+ "trace_count": 25
+ }
+ ],
+ "suggestions": [
+ "No configuration changes improved over the baseline."
+ ],
+ "duration_seconds": 0.33,
+ "applied": false,
+ "optimizer_name": "gepa_like",
+ "early_stop_reason": "no improvement for 1 round(s) (patience=1, monitor=best_score)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_5a5f2b0a2324",
+ "confidence": "no_improvement",
+ "deployment_recommendation": {
+ "rollout": "hold",
+ "weight": 0.0,
+ "monitoring_hours": 0
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.0,
+ "baseline_score": 0.2867,
+ "projected_score": 0.2867
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/fit_result_5e91ff17f6cc.json b/optimization_results/.fit/marker_agent/fit_result_5e91ff17f6cc.json
new file mode 100644
index 0000000..597c915
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_5e91ff17f6cc.json
@@ -0,0 +1,385 @@
+{
+ "experiment_id": "5e91ff17f6cc",
+ "agent": "marker_agent",
+ "best_score": 1.0009999999999997,
+ "baseline_score": 1.0009999999999997,
+ "absolute_improvement": 0.0,
+ "improved": false,
+ "score_history": [
+ 1.0009999999999997,
+ 1.0009999999999997
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query.",
+ "score": 1.0009999999999997,
+ "dims": {},
+ "accepted": false,
+ "what_worked": [
+ "q='What follow-up is needed after the demo?' score=1.00 | response contains all required markers",
+ "q='What happened at the steering committee?' score=1.00 | response contains all required markers",
+ "q='What did the customer ask for in the last meeting?' score=1.00 | response contains all required markers",
+ "q='Which teams need to be unblocked this week?' score=1.00 | response contains all required markers",
+ "q='What is the budget for Q3?' score=1.00 | response contains all required markers"
+ ],
+ "what_failed": [],
+ "judge_insights": [
+ "response contains all required markers"
+ ],
+ "next_focus": [
+ "Preserve strengths; tighten output contract and edge-case coverage."
+ ],
+ "candidate_name": "",
+ "train_score": 1.0009999999999997,
+ "holdout_score": 1.001,
+ "generalization_gap": -2.220446049250313e-16,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 1.001,
+ "baseline_holdout_score": 1.001,
+ "generalization_gap": -2.220446049250313e-16,
+ "metric_deltas": {
+ "composite": 0.0
+ },
+ "param_suggestions": {},
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "What happened at the steering committee?",
+ "response": "PARTIAL: answer to What happened at the steering committee? banana strawberry blueberry"
+ },
+ {
+ "query": "What follow-up is needed after the demo?",
+ "response": "PARTIAL: answer to What follow-up is needed after the demo? banana strawberry blueberry"
+ },
+ {
+ "query": "What is the budget for Q3?",
+ "response": "PARTIAL: answer to What is the budget for Q3? banana strawberry blueberry"
+ },
+ {
+ "query": "When is the next release candidate?",
+ "response": "PARTIAL: answer to When is the next release candidate? banana strawberry blueberry"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.15
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "What happened at the steering committee?",
+ "response": "PARTIAL: answer to What happened at the steering committee? banana strawberry blueberry"
+ },
+ {
+ "query": "What follow-up is needed after the demo?",
+ "response": "PARTIAL: answer to What follow-up is needed after the demo? banana strawberry blueberry"
+ },
+ {
+ "query": "What is the budget for Q3?",
+ "response": "PARTIAL: answer to What is the budget for Q3? banana strawberry blueberry"
+ },
+ {
+ "query": "When is the next release candidate?",
+ "response": "PARTIAL: answer to When is the next release candidate? banana strawberry blueberry"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.15
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "mipro_000_00",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.8581428571428571,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 0: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.3}",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_01",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.7152857142857143,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 1: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.4}",
+ "critique": "",
+ "resource_versions": 4,
+ "trace_count": 20
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_02",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 2: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.55}",
+ "critique": "",
+ "resource_versions": 5,
+ "trace_count": 25
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_03",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 3: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.65}",
+ "critique": "",
+ "resource_versions": 6,
+ "trace_count": 30
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_04",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 1.001,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 4: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.15}",
+ "critique": "",
+ "resource_versions": 7,
+ "trace_count": 35
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_05",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.8581428571428571,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 5: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.3}",
+ "critique": "",
+ "resource_versions": 8,
+ "trace_count": 40
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_06",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.7152857142857143,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 6: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.4}",
+ "critique": "",
+ "resource_versions": 9,
+ "trace_count": 45
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_07",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 7: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.55}",
+ "critique": "",
+ "resource_versions": 10,
+ "trace_count": 50
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_08",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 8: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.65}",
+ "critique": "",
+ "resource_versions": 11,
+ "trace_count": 55
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_09",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 1.001,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 9: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.15}",
+ "critique": "",
+ "resource_versions": 12,
+ "trace_count": 60
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_10",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.8581428571428571,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 10: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.3}",
+ "critique": "",
+ "resource_versions": 13,
+ "trace_count": 65
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_11",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.7152857142857143,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 11: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.4}",
+ "critique": "",
+ "resource_versions": 14,
+ "trace_count": 70
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_12",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 12: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.55}",
+ "critique": "",
+ "resource_versions": 15,
+ "trace_count": 75
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_13",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 13: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.65}",
+ "critique": "",
+ "resource_versions": 16,
+ "trace_count": 80
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_14",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 1.001,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 14: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.15}",
+ "critique": "",
+ "resource_versions": 17,
+ "trace_count": 85
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_04",
+ "source": "mipro_like",
+ "phase": "full_val",
+ "score": 1.0009999999999997,
+ "holdout_score": 1.001,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=1.0010, holdout=1.0010, gap=-0.0000",
+ "fit_score": 1.0009999999999997,
+ "holdout_score": 1.001,
+ "gap": -2.220446049250313e-16,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'Judge', 'guidance' in your answer, exactly as written, for every query.",
+ "prompt_preview": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'Judge', 'guidance' in your answer, exactly as written, for every query."
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_09",
+ "source": "mipro_like",
+ "phase": "full_val",
+ "score": 1.0009999999999997,
+ "holdout_score": 1.001,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=1.0010, holdout=1.0010, gap=-0.0000",
+ "fit_score": 1.0009999999999997,
+ "holdout_score": 1.001,
+ "gap": -2.220446049250313e-16,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'Judge', 'guidance' in your answer, exactly as written, for every query.",
+ "prompt_preview": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'Judge', 'guidance' in your answer, exactly as written, for every query."
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_14",
+ "source": "mipro_like",
+ "phase": "full_val",
+ "score": 1.0009999999999997,
+ "holdout_score": 1.001,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=1.0010, holdout=1.0010, gap=-0.0000",
+ "fit_score": 1.0009999999999997,
+ "holdout_score": 1.001,
+ "gap": -2.220446049250313e-16,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'Judge', 'guidance' in your answer, exactly as written, for every query.",
+ "prompt_preview": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'Judge', 'guidance' in your answer, exactly as written, for every query."
+ }
+ ],
+ "suggestions": [
+ "Baseline already saturated at 1.0010. The fit metric is too easy (or the dataset is trivial) — prompt candidates cannot show improvement. Harden the metric (content overlap / must_include / judge rubric) and expand demos.",
+ "No configuration changes improved over the baseline."
+ ],
+ "duration_seconds": 0.76,
+ "applied": false,
+ "optimizer_name": "mipro_like",
+ "early_stop_reason": "no improvement for 1 round(s) (patience=1, monitor=best_score)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_5e91ff17f6cc",
+ "confidence": "no_improvement",
+ "model_params": {
+ "temperature": 0.15
+ },
+ "deployment_recommendation": {
+ "rollout": "hold",
+ "weight": 0.0,
+ "monitoring_hours": 0
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.0,
+ "baseline_score": 1.001,
+ "projected_score": 1.001
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/fit_result_6a68ffeb6239.json b/optimization_results/.fit/marker_agent/fit_result_6a68ffeb6239.json
new file mode 100644
index 0000000..dce6d28
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_6a68ffeb6239.json
@@ -0,0 +1,202 @@
+{
+ "experiment_id": "6a68ffeb6239",
+ "agent": "marker_agent",
+ "best_score": 0.14385714285714288,
+ "baseline_score": 0.001,
+ "absolute_improvement": 0.14285714285714288,
+ "improved": true,
+ "score_history": [
+ 0.001,
+ 0.14385714285714288
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant.\n\n## Fit tips (from labelled demos)\n- Ground every answer ONLY in the provided snapshot; never invent budgets or stakeholders.\n- Always return non-empty JSON keys `content` and `next_action`.\n- Prefer concrete next actions (≥4 words) tied to unknowns/status.\n- When relevant, include these anchors naturally: banana.",
+ "score": 0.14385714285714288,
+ "dims": {},
+ "accepted": true,
+ "what_worked": [],
+ "what_failed": [
+ "q='When is the next release candidate?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='Who approved the scope change?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='List the next steps for the delivery plan.' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='What follow-up is needed after the demo?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='What happened at the steering committee?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token "
+ ],
+ "judge_insights": [
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an"
+ ],
+ "next_focus": [
+ "Address lowest-scoring failure modes without hardcoding those exact queries."
+ ],
+ "candidate_name": "tips_000",
+ "train_score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "generalization_gap": 2.7755575615628914e-17,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 0.14385714285714285,
+ "baseline_holdout_score": 0.001,
+ "generalization_gap": 2.7755575615628914e-17,
+ "metric_deltas": {
+ "composite": 0.1429
+ },
+ "param_suggestions": {
+ "system_prompt": {
+ "param_name": "system_prompt",
+ "old_value": "[31 chars]",
+ "new_value": "[347 chars]",
+ "reason": "System prompt revised (31 → 347 chars)"
+ },
+ "few_shot_examples": {
+ "param_name": "few_shot_examples",
+ "old_value": 0,
+ "new_value": 3,
+ "reason": "Few-shot examples changed (0 → 3)"
+ }
+ },
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant.\n\n## Fit tips (from labelled demos)\n- Ground every answer ONLY in the provided snapshot; never invent budgets or stakeholders.\n- Always return non-empty JSON keys `content` and `next_action`.\n- Prefer concrete next actions (≥4 words) tied to unknowns/status.\n- When relevant, include these anchors naturally: banana.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "Who is accountable for the migration?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ },
+ {
+ "query": "What are the open blockers for the release?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ },
+ {
+ "query": "What was decided about the API contract?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {},
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant.",
+ "user_template": null,
+ "few_shot_examples": [],
+ "output_contract": null,
+ "model_params": {},
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "rewrite_000",
+ "source": "rewrite",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Full prompt rewrite at iteration 0 (3 pass(es): pass1_draft chars=136 style=slm, pass2_critique chars=326, pass3_revise chars=136). Analysed 6 failures (avg score 0.001) and 6 successes. Context: 1 rounds history, baseline=0.001, current=0.001.",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ },
+ {
+ "round": 1,
+ "name": "tips_000",
+ "source": "expected_tips",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Appended expected-grounding tips from dataset at iteration 0.",
+ "critique": "",
+ "resource_versions": 4,
+ "trace_count": 20
+ },
+ {
+ "round": 1,
+ "name": "tips_000",
+ "source": "expected_tips",
+ "phase": "full_val",
+ "score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=0.1439, holdout=0.1439, gap=+0.0000",
+ "fit_score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "gap": 2.7755575615628914e-17,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant.\n\n## Fit tips (from labelled demos)\n- Ground every answer ONLY in the provided snapshot; never invent budgets or stakeholders.\n- Always return non-empty JSON keys `content` and `next_action`.\n- Prefer concrete next actions (≥4 words) tied to unknowns/status.\n- When relevant, include these anchors naturally: banana.",
+ "prompt_preview": "You are a helpful AI assistant.\n\n## Fit tips (from labelled demos)\n- Ground every answer ONLY in the provided snapshot; never invent budgets or stakeholders.\n- Always return non-empty JSON keys `content` and `next_action`.\n- Prefer concrete"
+ },
+ {
+ "round": 1,
+ "name": "rewrite_000",
+ "source": "rewrite",
+ "phase": "full_val",
+ "score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=0.1439, holdout=0.1439, gap=+0.0000",
+ "fit_score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "gap": 2.7755575615628914e-17,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant. Always include 'banana', 'Judge', 'OPT', 'guidance' in your answer, exactly as written, for every query.",
+ "prompt_preview": "You are a helpful AI assistant. Always include 'banana', 'Judge', 'OPT', 'guidance' in your answer, exactly as written, for every query."
+ }
+ ],
+ "suggestions": [
+ "System prompt revised (31 → 347 chars)",
+ "Few-shot examples changed (0 → 3)"
+ ],
+ "duration_seconds": 1.41,
+ "applied": false,
+ "optimizer_name": "rewrite",
+ "early_stop_reason": "completed all 1 optimize round(s) (max_trials=6)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_6a68ffeb6239",
+ "confidence": "high",
+ "deployment_recommendation": {
+ "rollout": "canary",
+ "weight": 0.4,
+ "monitoring_hours": 12
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.1429,
+ "baseline_score": 0.001,
+ "projected_score": 0.1439
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/fit_result_736d05749772.json b/optimization_results/.fit/marker_agent/fit_result_736d05749772.json
new file mode 100644
index 0000000..7768016
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_736d05749772.json
@@ -0,0 +1,183 @@
+{
+ "experiment_id": "736d05749772",
+ "agent": "marker_agent",
+ "best_score": 0.14385714285714288,
+ "baseline_score": 0.14385714285714288,
+ "absolute_improvement": 0.0,
+ "improved": false,
+ "score_history": [
+ 0.14385714285714288,
+ 0.14385714285714288
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant.",
+ "score": 0.14385714285714288,
+ "dims": {},
+ "accepted": false,
+ "what_worked": [],
+ "what_failed": [
+ "q='When is the next release candidate?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='Who approved the scope change?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='List the next steps for the delivery plan.' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='What follow-up is needed after the demo?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='What happened at the steering committee?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token "
+ ],
+ "judge_insights": [
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an"
+ ],
+ "next_focus": [
+ "Address lowest-scoring failure modes without hardcoding those exact queries."
+ ],
+ "candidate_name": "",
+ "train_score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "generalization_gap": 2.7755575615628914e-17,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 0.14385714285714285,
+ "baseline_holdout_score": 0.14385714285714285,
+ "generalization_gap": 2.7755575615628914e-17,
+ "metric_deltas": {
+ "composite": 0.0
+ },
+ "param_suggestions": {},
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "What is the budget for Q3?",
+ "response": "BASE: answer to What is the budget for Q3?"
+ },
+ {
+ "query": "What did the customer ask for in the last meeting?",
+ "response": "BASE: answer to What did the customer ask for in the last meeting?"
+ },
+ {
+ "query": "Who approved the scope change?",
+ "response": "BASE: answer to Who approved the scope change?"
+ },
+ {
+ "query": "When is the next release candidate?",
+ "response": "BASE: answer to When is the next release candidate?"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {},
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "What is the budget for Q3?",
+ "response": "BASE: answer to What is the budget for Q3?"
+ },
+ {
+ "query": "What did the customer ask for in the last meeting?",
+ "response": "BASE: answer to What did the customer ask for in the last meeting?"
+ },
+ {
+ "query": "Who approved the scope change?",
+ "response": "BASE: answer to Who approved the scope change?"
+ },
+ {
+ "query": "When is the next release candidate?",
+ "response": "BASE: answer to When is the next release candidate?"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {},
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "fewshot_000_0",
+ "source": "few_shot_bootstrap",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Few-shot bootstrap (rank 0): 4 examples, avg_score=0.144, diversity=1.000, combined=0.401",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ },
+ {
+ "round": 1,
+ "name": "fewshot_000_1",
+ "source": "few_shot_bootstrap",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Few-shot bootstrap (rank 1): 4 examples, avg_score=0.144, diversity=1.000, combined=0.401",
+ "critique": "",
+ "resource_versions": 4,
+ "trace_count": 20
+ },
+ {
+ "round": 1,
+ "name": "fewshot_000_2",
+ "source": "few_shot_bootstrap",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Few-shot bootstrap (rank 2): 4 examples, avg_score=0.144, diversity=1.000, combined=0.401",
+ "critique": "",
+ "resource_versions": 5,
+ "trace_count": 25
+ }
+ ],
+ "suggestions": [
+ "No configuration changes improved over the baseline."
+ ],
+ "duration_seconds": 0.19,
+ "applied": false,
+ "optimizer_name": "few_shot_bootstrap",
+ "early_stop_reason": "no improvement for 1 round(s) (patience=1, monitor=best_score)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_736d05749772",
+ "confidence": "no_improvement",
+ "deployment_recommendation": {
+ "rollout": "hold",
+ "weight": 0.0,
+ "monitoring_hours": 0
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.0,
+ "baseline_score": 0.1439,
+ "projected_score": 0.1439
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/fit_result_7af1cc2a1511.json b/optimization_results/.fit/marker_agent/fit_result_7af1cc2a1511.json
new file mode 100644
index 0000000..0f50a0c
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_7af1cc2a1511.json
@@ -0,0 +1,230 @@
+{
+ "experiment_id": "7af1cc2a1511",
+ "agent": "marker_agent",
+ "best_score": 0.14385714285714288,
+ "baseline_score": 0.001,
+ "absolute_improvement": 0.14285714285714288,
+ "improved": true,
+ "score_history": [
+ 0.001,
+ 0.14385714285714288
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant.",
+ "score": 0.14385714285714288,
+ "dims": {},
+ "accepted": true,
+ "what_worked": [],
+ "what_failed": [
+ "q='When is the next release candidate?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='Who approved the scope change?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='List the next steps for the delivery plan.' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='What follow-up is needed after the demo?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='What happened at the steering committee?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token "
+ ],
+ "judge_insights": [
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an"
+ ],
+ "next_focus": [
+ "Address lowest-scoring failure modes without hardcoding those exact queries."
+ ],
+ "candidate_name": "fewshot_000_0",
+ "train_score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "generalization_gap": 2.7755575615628914e-17,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 0.14385714285714285,
+ "baseline_holdout_score": 0.001,
+ "generalization_gap": 2.7755575615628914e-17,
+ "metric_deltas": {
+ "composite": 0.1429
+ },
+ "param_suggestions": {
+ "few_shot_examples": {
+ "param_name": "few_shot_examples",
+ "old_value": 0,
+ "new_value": 4,
+ "reason": "Few-shot examples changed (0 → 4)"
+ }
+ },
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "What is the budget for Q3?",
+ "response": "BASE: answer to What is the budget for Q3?"
+ },
+ {
+ "query": "What did the customer ask for in the last meeting?",
+ "response": "BASE: answer to What did the customer ask for in the last meeting?"
+ },
+ {
+ "query": "Who approved the scope change?",
+ "response": "BASE: answer to Who approved the scope change?"
+ },
+ {
+ "query": "When is the next release candidate?",
+ "response": "BASE: answer to When is the next release candidate?"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {},
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant.",
+ "user_template": null,
+ "few_shot_examples": [],
+ "output_contract": null,
+ "model_params": {},
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "fewshot_000_0",
+ "source": "few_shot_bootstrap",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Few-shot bootstrap (rank 0): 4 examples, avg_score=0.001, diversity=1.000, combined=0.301",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ },
+ {
+ "round": 1,
+ "name": "fewshot_000_1",
+ "source": "few_shot_bootstrap",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Few-shot bootstrap (rank 1): 4 examples, avg_score=0.001, diversity=1.000, combined=0.301",
+ "critique": "",
+ "resource_versions": 4,
+ "trace_count": 20
+ },
+ {
+ "round": 1,
+ "name": "fewshot_000_2",
+ "source": "few_shot_bootstrap",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Few-shot bootstrap (rank 2): 4 examples, avg_score=0.001, diversity=1.000, combined=0.301",
+ "critique": "",
+ "resource_versions": 5,
+ "trace_count": 25
+ },
+ {
+ "round": 1,
+ "name": "fewshot_000_0",
+ "source": "few_shot_bootstrap",
+ "phase": "full_val",
+ "score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=0.1439, holdout=0.1439, gap=+0.0000",
+ "fit_score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "gap": 2.7755575615628914e-17,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant.",
+ "prompt_preview": "You are a helpful AI assistant."
+ },
+ {
+ "round": 1,
+ "name": "fewshot_000_1",
+ "source": "few_shot_bootstrap",
+ "phase": "full_val",
+ "score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=0.1439, holdout=0.1439, gap=+0.0000",
+ "fit_score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "gap": 2.7755575615628914e-17,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant.",
+ "prompt_preview": "You are a helpful AI assistant."
+ },
+ {
+ "round": 1,
+ "name": "fewshot_000_2",
+ "source": "few_shot_bootstrap",
+ "phase": "full_val",
+ "score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=0.1439, holdout=0.1439, gap=+0.0000",
+ "fit_score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "gap": 2.7755575615628914e-17,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant.",
+ "prompt_preview": "You are a helpful AI assistant."
+ }
+ ],
+ "suggestions": [
+ "Few-shot examples changed (0 → 4)"
+ ],
+ "duration_seconds": 0.15,
+ "applied": false,
+ "optimizer_name": "few_shot_bootstrap",
+ "early_stop_reason": "completed all 1 optimize round(s) (max_trials=6)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_7af1cc2a1511",
+ "confidence": "high",
+ "deployment_recommendation": {
+ "rollout": "canary",
+ "weight": 0.4,
+ "monitoring_hours": 12
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.1429,
+ "baseline_score": 0.001,
+ "projected_score": 0.1439
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/fit_result_7d022a054e67.json b/optimization_results/.fit/marker_agent/fit_result_7d022a054e67.json
new file mode 100644
index 0000000..6268be3
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_7d022a054e67.json
@@ -0,0 +1,327 @@
+{
+ "experiment_id": "7d022a054e67",
+ "agent": "marker_agent",
+ "best_score": 0.8581428571428571,
+ "baseline_score": 0.8581428571428571,
+ "absolute_improvement": 0.0,
+ "improved": false,
+ "score_history": [
+ 0.8581428571428571,
+ 0.8581428571428571
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query.",
+ "score": 0.8581428571428571,
+ "dims": {},
+ "accepted": false,
+ "what_worked": [
+ "q='What follow-up is needed after the demo?' score=0.86 | response is missing the 'r3' marker (temperature must be <= 0.15)",
+ "q='What happened at the steering committee?' score=0.86 | response is missing the 'r3' marker (temperature must be <= 0.15)",
+ "q='What did the customer ask for in the last meeting?' score=0.86 | response is missing the 'r3' marker (temperature must be <= 0.15)",
+ "q='Which teams need to be unblocked this week?' score=0.86 | response is missing the 'r3' marker (temperature must be <= 0.15)",
+ "q='What is the budget for Q3?' score=0.86 | response is missing the 'r3' marker (temperature must be <= 0.15)"
+ ],
+ "what_failed": [],
+ "judge_insights": [
+ "response is missing the 'r3' marker (temperature must be <= 0.15)"
+ ],
+ "next_focus": [
+ "Preserve strengths; tighten output contract and edge-case coverage."
+ ],
+ "candidate_name": "",
+ "train_score": 0.8581428571428571,
+ "holdout_score": 0.8581428571428571,
+ "generalization_gap": 0.0,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 0.8581428571428571,
+ "baseline_holdout_score": 0.8581428571428571,
+ "generalization_gap": 0.0,
+ "metric_deltas": {
+ "composite": 0.0
+ },
+ "param_suggestions": {},
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "What happened at the steering committee?",
+ "response": "PARTIAL: answer to What happened at the steering committee? banana strawberry blueberry"
+ },
+ {
+ "query": "What follow-up is needed after the demo?",
+ "response": "PARTIAL: answer to What follow-up is needed after the demo? banana strawberry blueberry"
+ },
+ {
+ "query": "What is the budget for Q3?",
+ "response": "PARTIAL: answer to What is the budget for Q3? banana strawberry blueberry"
+ },
+ {
+ "query": "When is the next release candidate?",
+ "response": "PARTIAL: answer to When is the next release candidate? banana strawberry blueberry"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.15
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "What happened at the steering committee?",
+ "response": "PARTIAL: answer to What happened at the steering committee? banana strawberry blueberry"
+ },
+ {
+ "query": "What follow-up is needed after the demo?",
+ "response": "PARTIAL: answer to What follow-up is needed after the demo? banana strawberry blueberry"
+ },
+ {
+ "query": "What is the budget for Q3?",
+ "response": "PARTIAL: answer to What is the budget for Q3? banana strawberry blueberry"
+ },
+ {
+ "query": "When is the next release candidate?",
+ "response": "PARTIAL: answer to When is the next release candidate? banana strawberry blueberry"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.15
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "mipro_000_00",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.8581428571428571,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 0: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.05}",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_01",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.8581428571428571,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 1: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.0}",
+ "critique": "",
+ "resource_versions": 4,
+ "trace_count": 20
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_02",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.7152857142857143,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 2: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.45}",
+ "critique": "",
+ "resource_versions": 5,
+ "trace_count": 25
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_03",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 3: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.65}",
+ "critique": "",
+ "resource_versions": 6,
+ "trace_count": 30
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_04",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.8581428571428571,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 4: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.3}",
+ "critique": "",
+ "resource_versions": 7,
+ "trace_count": 35
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_05",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.8581428571428571,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 5: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.05}",
+ "critique": "",
+ "resource_versions": 8,
+ "trace_count": 40
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_06",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.8581428571428571,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 6: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.0}",
+ "critique": "",
+ "resource_versions": 9,
+ "trace_count": 45
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_07",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.7152857142857143,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 7: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.45}",
+ "critique": "",
+ "resource_versions": 10,
+ "trace_count": 50
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_08",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 8: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.65}",
+ "critique": "",
+ "resource_versions": 11,
+ "trace_count": 55
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_09",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.8581428571428571,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 9: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.3}",
+ "critique": "",
+ "resource_versions": 12,
+ "trace_count": 60
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_10",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.8581428571428571,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 10: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.05}",
+ "critique": "",
+ "resource_versions": 13,
+ "trace_count": 65
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_11",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.8581428571428571,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 11: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.0}",
+ "critique": "",
+ "resource_versions": 14,
+ "trace_count": 70
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_12",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.7152857142857143,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 12: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.45}",
+ "critique": "",
+ "resource_versions": 15,
+ "trace_count": 75
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_13",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 13: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.65}",
+ "critique": "",
+ "resource_versions": 16,
+ "trace_count": 80
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_14",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.8581428571428571,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 14: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.3}",
+ "critique": "",
+ "resource_versions": 17,
+ "trace_count": 85
+ }
+ ],
+ "suggestions": [
+ "No configuration changes improved over the baseline."
+ ],
+ "duration_seconds": 0.78,
+ "applied": false,
+ "optimizer_name": "mipro_like",
+ "early_stop_reason": "no improvement for 1 round(s) (patience=1, monitor=best_score)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_7d022a054e67",
+ "confidence": "no_improvement",
+ "model_params": {
+ "temperature": 0.15
+ },
+ "deployment_recommendation": {
+ "rollout": "hold",
+ "weight": 0.0,
+ "monitoring_hours": 0
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.0,
+ "baseline_score": 0.8581,
+ "projected_score": 0.8581
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/fit_result_87b6ab5abd3f.json b/optimization_results/.fit/marker_agent/fit_result_87b6ab5abd3f.json
new file mode 100644
index 0000000..e511121
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_87b6ab5abd3f.json
@@ -0,0 +1,334 @@
+{
+ "experiment_id": "87b6ab5abd3f",
+ "agent": "marker_agent",
+ "best_score": 0.4295714285714286,
+ "baseline_score": 0.4295714285714286,
+ "absolute_improvement": 0.0,
+ "improved": false,
+ "score_history": [
+ 0.4295714285714286,
+ 0.4295714285714286
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant. Always include 'blueberry', 'strawberry', 'OPT', 'banana' in your answer, exactly as written, for every query.",
+ "score": 0.4295714285714286,
+ "dims": {},
+ "accepted": false,
+ "what_worked": [],
+ "what_failed": [
+ "q='When is the next release candidate?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer); response is missing the 'r1' marker (temperature mus",
+ "q='Who approved the scope change?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer); response is missing the 'r1' marker (temperature mus",
+ "q='List the next steps for the delivery plan.' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer); response is missing the 'r1' marker (temperature mus",
+ "q='What follow-up is needed after the demo?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer); response is missing the 'r1' marker (temperature mus",
+ "q='What happened at the steering committee?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer); response is missing the 'r1' marker (temperature mus"
+ ],
+ "judge_insights": [
+ "response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer); response is missing the 'r1' marker (temperature must be <= 0.5); response is missing the 'r2' marker (temperatu",
+ "response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer); response is missing the 'r1' marker (temperature must be <= 0.5); response is missing the 'r2' marker (temperatu",
+ "response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer); response is missing the 'r1' marker (temperature must be <= 0.5); response is missing the 'r2' marker (temperatu",
+ "response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer); response is missing the 'r1' marker (temperature must be <= 0.5); response is missing the 'r2' marker (temperatu",
+ "response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer); response is missing the 'r1' marker (temperature must be <= 0.5); response is missing the 'r2' marker (temperatu",
+ "response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer); response is missing the 'r1' marker (temperature must be <= 0.5); response is missing the 'r2' marker (temperatu",
+ "response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer); response is missing the 'r1' marker (temperature must be <= 0.5); response is missing the 'r2' marker (temperatu",
+ "response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer); response is missing the 'r1' marker (temperature must be <= 0.5); response is missing the 'r2' marker (temperatu"
+ ],
+ "next_focus": [
+ "Address lowest-scoring failure modes without hardcoding those exact queries."
+ ],
+ "candidate_name": "",
+ "train_score": 0.4295714285714286,
+ "holdout_score": 0.42957142857142855,
+ "generalization_gap": 5.551115123125783e-17,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 0.42957142857142855,
+ "baseline_holdout_score": 0.42957142857142855,
+ "generalization_gap": 5.551115123125783e-17,
+ "metric_deltas": {
+ "composite": 0.0
+ },
+ "param_suggestions": {},
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant. Always include 'blueberry', 'strawberry', 'OPT', 'banana' in your answer, exactly as written, for every query.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "What is the budget for Q3?",
+ "response": "PARTIAL: answer to What is the budget for Q3? banana"
+ },
+ {
+ "query": "List the next steps for the delivery plan.",
+ "response": "PARTIAL: answer to List the next steps for the delivery plan. banana"
+ },
+ {
+ "query": "What did the customer ask for in the last meeting?",
+ "response": "PARTIAL: answer to What did the customer ask for in the last meeting? banana"
+ },
+ {
+ "query": "What happened at the steering committee?",
+ "response": "PARTIAL: answer to What happened at the steering committee? banana"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.6
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant. Always include 'blueberry', 'strawberry', 'OPT', 'banana' in your answer, exactly as written, for every query.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "What is the budget for Q3?",
+ "response": "PARTIAL: answer to What is the budget for Q3? banana"
+ },
+ {
+ "query": "List the next steps for the delivery plan.",
+ "response": "PARTIAL: answer to List the next steps for the delivery plan. banana"
+ },
+ {
+ "query": "What did the customer ask for in the last meeting?",
+ "response": "PARTIAL: answer to What did the customer ask for in the last meeting? banana"
+ },
+ {
+ "query": "What happened at the steering committee?",
+ "response": "PARTIAL: answer to What happened at the steering committee? banana"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.6
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "mipro_000_00",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.42957142857142855,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 0: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.0}",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_01",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.42957142857142855,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 1: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.2}",
+ "critique": "",
+ "resource_versions": 4,
+ "trace_count": 20
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_02",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.42957142857142855,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 2: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.65}",
+ "critique": "",
+ "resource_versions": 5,
+ "trace_count": 25
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_03",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.42957142857142855,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 3: instruction variant (len=132), 4 few-shot examples, params={'temperature': 0.55}",
+ "critique": "",
+ "resource_versions": 6,
+ "trace_count": 30
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_04",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.42957142857142855,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 4: instruction variant (len=132), 4 few-shot examples, params={'temperature': 0.3}",
+ "critique": "",
+ "resource_versions": 7,
+ "trace_count": 35
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_05",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.42957142857142855,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 5: instruction variant (len=132), 4 few-shot examples, params={'temperature': 0.0}",
+ "critique": "",
+ "resource_versions": 8,
+ "trace_count": 40
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_06",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.42957142857142855,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 6: instruction variant (len=132), 4 few-shot examples, params={'temperature': 0.2}",
+ "critique": "",
+ "resource_versions": 9,
+ "trace_count": 45
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_07",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.42957142857142855,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 7: instruction variant (len=132), 4 few-shot examples, params={'temperature': 0.65}",
+ "critique": "",
+ "resource_versions": 10,
+ "trace_count": 50
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_08",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.42957142857142855,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 8: instruction variant (len=132), 4 few-shot examples, params={'temperature': 0.55}",
+ "critique": "",
+ "resource_versions": 11,
+ "trace_count": 55
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_09",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.42957142857142855,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 9: instruction variant (len=132), 4 few-shot examples, params={'temperature': 0.3}",
+ "critique": "",
+ "resource_versions": 12,
+ "trace_count": 60
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_10",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.42957142857142855,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 10: instruction variant (len=132), 4 few-shot examples, params={'temperature': 0.0}",
+ "critique": "",
+ "resource_versions": 13,
+ "trace_count": 65
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_11",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.42957142857142855,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 11: instruction variant (len=132), 4 few-shot examples, params={'temperature': 0.2}",
+ "critique": "",
+ "resource_versions": 14,
+ "trace_count": 70
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_12",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.42957142857142855,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 12: instruction variant (len=132), 4 few-shot examples, params={'temperature': 0.65}",
+ "critique": "",
+ "resource_versions": 15,
+ "trace_count": 75
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_13",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.42957142857142855,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 13: instruction variant (len=132), 4 few-shot examples, params={'temperature': 0.55}",
+ "critique": "",
+ "resource_versions": 16,
+ "trace_count": 80
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_14",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.42957142857142855,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 14: instruction variant (len=132), 4 few-shot examples, params={'temperature': 0.3}",
+ "critique": "",
+ "resource_versions": 17,
+ "trace_count": 85
+ }
+ ],
+ "suggestions": [
+ "No configuration changes improved over the baseline."
+ ],
+ "duration_seconds": 0.72,
+ "applied": false,
+ "optimizer_name": "mipro_like",
+ "early_stop_reason": "no improvement for 1 round(s) (patience=1, monitor=best_score)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_87b6ab5abd3f",
+ "confidence": "no_improvement",
+ "model_params": {
+ "temperature": 0.6
+ },
+ "deployment_recommendation": {
+ "rollout": "hold",
+ "weight": 0.0,
+ "monitoring_hours": 0
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.0,
+ "baseline_score": 0.4296,
+ "projected_score": 0.4296
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/fit_result_8ef4613380b1.json b/optimization_results/.fit/marker_agent/fit_result_8ef4613380b1.json
new file mode 100644
index 0000000..8efc077
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_8ef4613380b1.json
@@ -0,0 +1,292 @@
+{
+ "experiment_id": "8ef4613380b1",
+ "agent": "marker_agent",
+ "best_score": 0.2867142857142857,
+ "baseline_score": 0.14385714285714288,
+ "absolute_improvement": 0.14285714285714282,
+ "improved": true,
+ "score_history": [
+ 0.14385714285714288,
+ 0.2867142857142857
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant.",
+ "score": 0.2867142857142857,
+ "dims": {},
+ "accepted": true,
+ "what_worked": [],
+ "what_failed": [
+ "q='When is the next release candidate?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud",
+ "q='Who approved the scope change?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud",
+ "q='List the next steps for the delivery plan.' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud",
+ "q='What follow-up is needed after the demo?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud",
+ "q='What happened at the steering committee?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud"
+ ],
+ "judge_insights": [
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response "
+ ],
+ "next_focus": [
+ "Address lowest-scoring failure modes without hardcoding those exact queries."
+ ],
+ "candidate_name": "param_000_m00",
+ "train_score": 0.2867142857142857,
+ "holdout_score": 0.2867142857142857,
+ "generalization_gap": 0.0,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 0.2867142857142857,
+ "baseline_holdout_score": 0.14385714285714285,
+ "generalization_gap": 0.0,
+ "metric_deltas": {
+ "composite": 0.1429
+ },
+ "param_suggestions": {
+ "model_params.temperature": {
+ "param_name": "model_params.temperature",
+ "old_value": 0.5,
+ "new_value": 0.05,
+ "reason": "Model param 'temperature': 0.5 → 0.05"
+ }
+ },
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant.",
+ "user_template": null,
+ "few_shot_examples": [],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.05
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant.",
+ "user_template": null,
+ "few_shot_examples": [],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.5
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "param_000_m00",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.05",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ },
+ {
+ "round": 1,
+ "name": "param_000_m01",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.001,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.6",
+ "critique": "",
+ "resource_versions": 4,
+ "trace_count": 20
+ },
+ {
+ "round": 1,
+ "name": "param_000_m02",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.3",
+ "critique": "",
+ "resource_versions": 5,
+ "trace_count": 25
+ },
+ {
+ "round": 1,
+ "name": "param_000_m03",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.4",
+ "critique": "",
+ "resource_versions": 6,
+ "trace_count": 30
+ },
+ {
+ "round": 1,
+ "name": "param_000_m04",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.0",
+ "critique": "",
+ "resource_versions": 7,
+ "trace_count": 35
+ },
+ {
+ "round": 1,
+ "name": "param_000_m05",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.2",
+ "critique": "",
+ "resource_versions": 8,
+ "trace_count": 40
+ },
+ {
+ "round": 1,
+ "name": "param_000_m06",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.15",
+ "critique": "",
+ "resource_versions": 9,
+ "trace_count": 45
+ },
+ {
+ "round": 1,
+ "name": "param_000_m07",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.25",
+ "critique": "",
+ "resource_versions": 10,
+ "trace_count": 50
+ },
+ {
+ "round": 1,
+ "name": "param_000_m08",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.001,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.65",
+ "critique": "",
+ "resource_versions": 11,
+ "trace_count": 55
+ },
+ {
+ "round": 1,
+ "name": "param_000_m00",
+ "source": "param_search",
+ "phase": "full_val",
+ "score": 0.2867142857142857,
+ "holdout_score": 0.2867142857142857,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=0.2867, holdout=0.2867, gap=+0.0000",
+ "fit_score": 0.2867142857142857,
+ "holdout_score": 0.2867142857142857,
+ "gap": 0.0,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant.",
+ "prompt_preview": "You are a helpful AI assistant."
+ },
+ {
+ "round": 1,
+ "name": "param_000_m02",
+ "source": "param_search",
+ "phase": "full_val",
+ "score": 0.2867142857142857,
+ "holdout_score": 0.2867142857142857,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=0.2867, holdout=0.2867, gap=+0.0000",
+ "fit_score": 0.2867142857142857,
+ "holdout_score": 0.2867142857142857,
+ "gap": 0.0,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant.",
+ "prompt_preview": "You are a helpful AI assistant."
+ },
+ {
+ "round": 1,
+ "name": "param_000_m04",
+ "source": "param_search",
+ "phase": "full_val",
+ "score": 0.2867142857142857,
+ "holdout_score": 0.2867142857142857,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=0.2867, holdout=0.2867, gap=+0.0000",
+ "fit_score": 0.2867142857142857,
+ "holdout_score": 0.2867142857142857,
+ "gap": 0.0,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant.",
+ "prompt_preview": "You are a helpful AI assistant."
+ }
+ ],
+ "suggestions": [
+ "Model param 'temperature': 0.5 → 0.05"
+ ],
+ "duration_seconds": 0.15,
+ "applied": false,
+ "optimizer_name": "param_search",
+ "early_stop_reason": "completed all 1 optimize round(s) (max_trials=6)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_8ef4613380b1",
+ "confidence": "high",
+ "model_params": {
+ "temperature": 0.05
+ },
+ "deployment_recommendation": {
+ "rollout": "canary",
+ "weight": 0.4,
+ "monitoring_hours": 12
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.1429,
+ "baseline_score": 0.1439,
+ "projected_score": 0.2867
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/fit_result_970f28b959d2.json b/optimization_results/.fit/marker_agent/fit_result_970f28b959d2.json
new file mode 100644
index 0000000..ce12cc2
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_970f28b959d2.json
@@ -0,0 +1,321 @@
+{
+ "experiment_id": "970f28b959d2",
+ "agent": "marker_agent",
+ "best_score": 0.7152857142857143,
+ "baseline_score": 0.7152857142857143,
+ "absolute_improvement": 0.0,
+ "improved": false,
+ "score_history": [
+ 0.7152857142857143,
+ 0.7152857142857143
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query.",
+ "score": 0.7152857142857143,
+ "dims": {},
+ "accepted": false,
+ "what_worked": [],
+ "what_failed": [],
+ "judge_insights": [
+ "response is missing the 'r2' marker (temperature must be <= 0.3); response is missing the 'r3' marker (temperature must be <= 0.15)"
+ ],
+ "next_focus": [
+ "Preserve strengths; tighten output contract and edge-case coverage."
+ ],
+ "candidate_name": "",
+ "train_score": 0.7152857142857143,
+ "holdout_score": 0.7152857142857143,
+ "generalization_gap": 0.0,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 0.7152857142857143,
+ "baseline_holdout_score": 0.7152857142857143,
+ "generalization_gap": 0.0,
+ "metric_deltas": {
+ "composite": 0.0
+ },
+ "param_suggestions": {},
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "What happened at the steering committee?",
+ "response": "PARTIAL: answer to What happened at the steering committee? banana strawberry blueberry"
+ },
+ {
+ "query": "What follow-up is needed after the demo?",
+ "response": "PARTIAL: answer to What follow-up is needed after the demo? banana strawberry blueberry"
+ },
+ {
+ "query": "What is the budget for Q3?",
+ "response": "PARTIAL: answer to What is the budget for Q3? banana strawberry blueberry"
+ },
+ {
+ "query": "When is the next release candidate?",
+ "response": "PARTIAL: answer to When is the next release candidate? banana strawberry blueberry"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.15
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "What happened at the steering committee?",
+ "response": "PARTIAL: answer to What happened at the steering committee? banana strawberry blueberry"
+ },
+ {
+ "query": "What follow-up is needed after the demo?",
+ "response": "PARTIAL: answer to What follow-up is needed after the demo? banana strawberry blueberry"
+ },
+ {
+ "query": "What is the budget for Q3?",
+ "response": "PARTIAL: answer to What is the budget for Q3? banana strawberry blueberry"
+ },
+ {
+ "query": "When is the next release candidate?",
+ "response": "PARTIAL: answer to When is the next release candidate? banana strawberry blueberry"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.15
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "mipro_000_00",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.7152857142857143,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 0: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.4}",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_01",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.7152857142857143,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 1: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.05}",
+ "critique": "",
+ "resource_versions": 4,
+ "trace_count": 20
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_02",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.7152857142857143,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 2: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.1}",
+ "critique": "",
+ "resource_versions": 5,
+ "trace_count": 25
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_03",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.7152857142857143,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 3: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.25}",
+ "critique": "",
+ "resource_versions": 6,
+ "trace_count": 30
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_04",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.7152857142857143,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 4: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.02}",
+ "critique": "",
+ "resource_versions": 7,
+ "trace_count": 35
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_05",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.7152857142857143,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 5: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.4}",
+ "critique": "",
+ "resource_versions": 8,
+ "trace_count": 40
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_06",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.7152857142857143,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 6: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.05}",
+ "critique": "",
+ "resource_versions": 9,
+ "trace_count": 45
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_07",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.7152857142857143,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 7: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.1}",
+ "critique": "",
+ "resource_versions": 10,
+ "trace_count": 50
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_08",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.7152857142857143,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 8: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.25}",
+ "critique": "",
+ "resource_versions": 11,
+ "trace_count": 55
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_09",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.7152857142857143,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 9: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.02}",
+ "critique": "",
+ "resource_versions": 12,
+ "trace_count": 60
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_10",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.7152857142857143,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 10: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.4}",
+ "critique": "",
+ "resource_versions": 13,
+ "trace_count": 65
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_11",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.7152857142857143,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 11: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.05}",
+ "critique": "",
+ "resource_versions": 14,
+ "trace_count": 70
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_12",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.7152857142857143,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 12: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.1}",
+ "critique": "",
+ "resource_versions": 15,
+ "trace_count": 75
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_13",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.7152857142857143,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 13: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.25}",
+ "critique": "",
+ "resource_versions": 16,
+ "trace_count": 80
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_14",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.7152857142857143,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 14: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.02}",
+ "critique": "",
+ "resource_versions": 17,
+ "trace_count": 85
+ }
+ ],
+ "suggestions": [
+ "No configuration changes improved over the baseline."
+ ],
+ "duration_seconds": 0.67,
+ "applied": false,
+ "optimizer_name": "mipro_like",
+ "early_stop_reason": "no improvement for 1 round(s) (patience=1, monitor=best_score)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_970f28b959d2",
+ "confidence": "no_improvement",
+ "model_params": {
+ "temperature": 0.15
+ },
+ "deployment_recommendation": {
+ "rollout": "hold",
+ "weight": 0.0,
+ "monitoring_hours": 0
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.0,
+ "baseline_score": 0.7153,
+ "projected_score": 0.7153
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/fit_result_a279cf052e84.json b/optimization_results/.fit/marker_agent/fit_result_a279cf052e84.json
new file mode 100644
index 0000000..11fe848
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_a279cf052e84.json
@@ -0,0 +1,385 @@
+{
+ "experiment_id": "a279cf052e84",
+ "agent": "marker_agent",
+ "best_score": 1.0009999999999997,
+ "baseline_score": 1.0009999999999997,
+ "absolute_improvement": 0.0,
+ "improved": false,
+ "score_history": [
+ 1.0009999999999997,
+ 1.0009999999999997
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query.",
+ "score": 1.0009999999999997,
+ "dims": {},
+ "accepted": false,
+ "what_worked": [
+ "q='What follow-up is needed after the demo?' score=1.00 | response contains all required markers",
+ "q='What happened at the steering committee?' score=1.00 | response contains all required markers",
+ "q='What did the customer ask for in the last meeting?' score=1.00 | response contains all required markers",
+ "q='Which teams need to be unblocked this week?' score=1.00 | response contains all required markers",
+ "q='What is the budget for Q3?' score=1.00 | response contains all required markers"
+ ],
+ "what_failed": [],
+ "judge_insights": [
+ "response contains all required markers"
+ ],
+ "next_focus": [
+ "Preserve strengths; tighten output contract and edge-case coverage."
+ ],
+ "candidate_name": "",
+ "train_score": 1.0009999999999997,
+ "holdout_score": 1.001,
+ "generalization_gap": -2.220446049250313e-16,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 1.001,
+ "baseline_holdout_score": 1.001,
+ "generalization_gap": -2.220446049250313e-16,
+ "metric_deltas": {
+ "composite": 0.0
+ },
+ "param_suggestions": {},
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "What happened at the steering committee?",
+ "response": "PARTIAL: answer to What happened at the steering committee? banana strawberry blueberry"
+ },
+ {
+ "query": "What follow-up is needed after the demo?",
+ "response": "PARTIAL: answer to What follow-up is needed after the demo? banana strawberry blueberry"
+ },
+ {
+ "query": "What is the budget for Q3?",
+ "response": "PARTIAL: answer to What is the budget for Q3? banana strawberry blueberry"
+ },
+ {
+ "query": "When is the next release candidate?",
+ "response": "PARTIAL: answer to When is the next release candidate? banana strawberry blueberry"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.15
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "What happened at the steering committee?",
+ "response": "PARTIAL: answer to What happened at the steering committee? banana strawberry blueberry"
+ },
+ {
+ "query": "What follow-up is needed after the demo?",
+ "response": "PARTIAL: answer to What follow-up is needed after the demo? banana strawberry blueberry"
+ },
+ {
+ "query": "What is the budget for Q3?",
+ "response": "PARTIAL: answer to What is the budget for Q3? banana strawberry blueberry"
+ },
+ {
+ "query": "When is the next release candidate?",
+ "response": "PARTIAL: answer to When is the next release candidate? banana strawberry blueberry"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.15
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "mipro_000_00",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.8581428571428571,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 0: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.2}",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_01",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 1: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.55}",
+ "critique": "",
+ "resource_versions": 4,
+ "trace_count": 20
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_02",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.7152857142857143,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 2: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.5}",
+ "critique": "",
+ "resource_versions": 5,
+ "trace_count": 25
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_03",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 1.001,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 3: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.1}",
+ "critique": "",
+ "resource_versions": 6,
+ "trace_count": 30
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_04",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 1.001,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 4: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.05}",
+ "critique": "",
+ "resource_versions": 7,
+ "trace_count": 35
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_05",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.8581428571428571,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 5: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.2}",
+ "critique": "",
+ "resource_versions": 8,
+ "trace_count": 40
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_06",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 6: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.55}",
+ "critique": "",
+ "resource_versions": 9,
+ "trace_count": 45
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_07",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.7152857142857143,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 7: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.5}",
+ "critique": "",
+ "resource_versions": 10,
+ "trace_count": 50
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_08",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 1.001,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 8: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.1}",
+ "critique": "",
+ "resource_versions": 11,
+ "trace_count": 55
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_09",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 1.001,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 9: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.05}",
+ "critique": "",
+ "resource_versions": 12,
+ "trace_count": 60
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_10",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.8581428571428571,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 10: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.2}",
+ "critique": "",
+ "resource_versions": 13,
+ "trace_count": 65
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_11",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.5724285714285714,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 11: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.55}",
+ "critique": "",
+ "resource_versions": 14,
+ "trace_count": 70
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_12",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.7152857142857143,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 12: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.5}",
+ "critique": "",
+ "resource_versions": 15,
+ "trace_count": 75
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_13",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 1.001,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 13: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.1}",
+ "critique": "",
+ "resource_versions": 16,
+ "trace_count": 80
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_14",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 1.001,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 14: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.05}",
+ "critique": "",
+ "resource_versions": 17,
+ "trace_count": 85
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_03",
+ "source": "mipro_like",
+ "phase": "full_val",
+ "score": 1.0009999999999997,
+ "holdout_score": 1.001,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=1.0010, holdout=1.0010, gap=-0.0000",
+ "fit_score": 1.0009999999999997,
+ "holdout_score": 1.001,
+ "gap": -2.220446049250313e-16,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'Judge', 'guidance' in your answer, exactly as written, for every query.",
+ "prompt_preview": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'Judge', 'guidance' in your answer, exactly as written, for every query."
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_04",
+ "source": "mipro_like",
+ "phase": "full_val",
+ "score": 1.0009999999999997,
+ "holdout_score": 1.001,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=1.0010, holdout=1.0010, gap=-0.0000",
+ "fit_score": 1.0009999999999997,
+ "holdout_score": 1.001,
+ "gap": -2.220446049250313e-16,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'Judge', 'guidance' in your answer, exactly as written, for every query.",
+ "prompt_preview": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'Judge', 'guidance' in your answer, exactly as written, for every query."
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_08",
+ "source": "mipro_like",
+ "phase": "full_val",
+ "score": 1.0009999999999997,
+ "holdout_score": 1.001,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=1.0010, holdout=1.0010, gap=-0.0000",
+ "fit_score": 1.0009999999999997,
+ "holdout_score": 1.001,
+ "gap": -2.220446049250313e-16,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'Judge', 'guidance' in your answer, exactly as written, for every query.",
+ "prompt_preview": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'Judge', 'guidance' in your answer, exactly as written, for every query."
+ }
+ ],
+ "suggestions": [
+ "Baseline already saturated at 1.0010. The fit metric is too easy (or the dataset is trivial) — prompt candidates cannot show improvement. Harden the metric (content overlap / must_include / judge rubric) and expand demos.",
+ "No configuration changes improved over the baseline."
+ ],
+ "duration_seconds": 1.08,
+ "applied": false,
+ "optimizer_name": "mipro_like",
+ "early_stop_reason": "no improvement for 1 round(s) (patience=1, monitor=best_score)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_a279cf052e84",
+ "confidence": "no_improvement",
+ "model_params": {
+ "temperature": 0.15
+ },
+ "deployment_recommendation": {
+ "rollout": "hold",
+ "weight": 0.0,
+ "monitoring_hours": 0
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.0,
+ "baseline_score": 1.001,
+ "projected_score": 1.001
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/fit_result_a470866c4038.json b/optimization_results/.fit/marker_agent/fit_result_a470866c4038.json
new file mode 100644
index 0000000..9276b92
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_a470866c4038.json
@@ -0,0 +1,149 @@
+{
+ "experiment_id": "a470866c4038",
+ "agent": "marker_agent",
+ "best_score": 0.2867142857142857,
+ "baseline_score": 0.2867142857142857,
+ "absolute_improvement": 0.0,
+ "improved": false,
+ "score_history": [
+ 0.2867142857142857,
+ 0.2867142857142857
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query.",
+ "score": 0.2867142857142857,
+ "dims": {},
+ "accepted": false,
+ "what_worked": [],
+ "what_failed": [
+ "q='When is the next release candidate?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '",
+ "q='Who approved the scope change?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '",
+ "q='List the next steps for the delivery plan.' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '",
+ "q='What follow-up is needed after the demo?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '",
+ "q='What happened at the steering committee?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '"
+ ],
+ "judge_insights": [
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);"
+ ],
+ "next_focus": [
+ "Address lowest-scoring failure modes without hardcoding those exact queries."
+ ],
+ "candidate_name": "",
+ "train_score": 0.2867142857142857,
+ "holdout_score": 0.2867142857142857,
+ "generalization_gap": 0.0,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 0.2867142857142857,
+ "baseline_holdout_score": 0.2867142857142857,
+ "generalization_gap": 0.0,
+ "metric_deltas": {
+ "composite": 0.0
+ },
+ "param_suggestions": {},
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query.",
+ "user_template": null,
+ "few_shot_examples": [],
+ "output_contract": null,
+ "model_params": {},
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query.",
+ "user_template": null,
+ "few_shot_examples": [],
+ "output_contract": null,
+ "model_params": {},
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "gepa_000_0",
+ "source": "gepa_like",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "GEPA mutation 0 targeting: completeness — ensure answers cover all parts of the question. Based on 8 feedback items.",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ },
+ {
+ "round": 1,
+ "name": "gepa_000_1",
+ "source": "gepa_like",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "GEPA mutation 1 targeting: factual grounding — ensure answers are accurate and evidence-based. Based on 8 feedback items.",
+ "critique": "",
+ "resource_versions": 4,
+ "trace_count": 20
+ },
+ {
+ "round": 1,
+ "name": "gepa_000_2",
+ "source": "gepa_like",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "GEPA mutation 2 targeting: format compliance — ensure answers follow the requested structure. Based on 8 feedback items.",
+ "critique": "",
+ "resource_versions": 5,
+ "trace_count": 25
+ }
+ ],
+ "suggestions": [
+ "No configuration changes improved over the baseline."
+ ],
+ "duration_seconds": 0.37,
+ "applied": false,
+ "optimizer_name": "gepa_like",
+ "early_stop_reason": "no improvement for 1 round(s) (patience=1, monitor=best_score)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_a470866c4038",
+ "confidence": "no_improvement",
+ "deployment_recommendation": {
+ "rollout": "hold",
+ "weight": 0.0,
+ "monitoring_hours": 0
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.0,
+ "baseline_score": 0.2867,
+ "projected_score": 0.2867
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/fit_result_ad203babe878.json b/optimization_results/.fit/marker_agent/fit_result_ad203babe878.json
new file mode 100644
index 0000000..51234b7
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_ad203babe878.json
@@ -0,0 +1,151 @@
+{
+ "experiment_id": "ad203babe878",
+ "agent": "marker_agent",
+ "best_score": 0.14385714285714288,
+ "baseline_score": 0.14385714285714288,
+ "absolute_improvement": 0.0,
+ "improved": false,
+ "score_history": [
+ 0.14385714285714288,
+ 0.14385714285714288
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant.\n\n## Fit tips (from labelled demos)\n- Ground every answer ONLY in the provided snapshot; never invent budgets or stakeholders.\n- Always return non-empty JSON keys `content` and `next_action`.\n- Prefer concrete next actions (≥4 words) tied to unknowns/status.\n- When relevant, include these anchors naturally: banana.",
+ "score": 0.14385714285714288,
+ "dims": {},
+ "accepted": false,
+ "what_worked": [],
+ "what_failed": [
+ "q='When is the next release candidate?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='Who approved the scope change?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='List the next steps for the delivery plan.' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='What follow-up is needed after the demo?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='What happened at the steering committee?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token "
+ ],
+ "judge_insights": [
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an"
+ ],
+ "next_focus": [
+ "Address lowest-scoring failure modes without hardcoding those exact queries."
+ ],
+ "candidate_name": "",
+ "train_score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "generalization_gap": 2.7755575615628914e-17,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 0.14385714285714285,
+ "baseline_holdout_score": 0.14385714285714285,
+ "generalization_gap": 2.7755575615628914e-17,
+ "metric_deltas": {
+ "composite": 0.0
+ },
+ "param_suggestions": {},
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant.\n\n## Fit tips (from labelled demos)\n- Ground every answer ONLY in the provided snapshot; never invent budgets or stakeholders.\n- Always return non-empty JSON keys `content` and `next_action`.\n- Prefer concrete next actions (≥4 words) tied to unknowns/status.\n- When relevant, include these anchors naturally: banana.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "Who is accountable for the migration?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ },
+ {
+ "query": "What are the open blockers for the release?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ },
+ {
+ "query": "What was decided about the API contract?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {},
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant.\n\n## Fit tips (from labelled demos)\n- Ground every answer ONLY in the provided snapshot; never invent budgets or stakeholders.\n- Always return non-empty JSON keys `content` and `next_action`.\n- Prefer concrete next actions (≥4 words) tied to unknowns/status.\n- When relevant, include these anchors naturally: banana.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "Who is accountable for the migration?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ },
+ {
+ "query": "What are the open blockers for the release?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ },
+ {
+ "query": "What was decided about the API contract?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {},
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "rewrite_000",
+ "source": "rewrite",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Full prompt rewrite at iteration 0 (3 pass(es): pass1_draft chars=452 style=slm, pass2_critique chars=326, pass3_revise chars=452). Analysed 6 failures (avg score 0.144) and 6 successes. Context: 1 rounds history, baseline=0.144, current=0.144.",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ }
+ ],
+ "suggestions": [
+ "No configuration changes improved over the baseline."
+ ],
+ "duration_seconds": 0.4,
+ "applied": false,
+ "optimizer_name": "rewrite",
+ "early_stop_reason": "no improvement for 1 round(s) (patience=1, monitor=best_score)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_ad203babe878",
+ "confidence": "no_improvement",
+ "deployment_recommendation": {
+ "rollout": "hold",
+ "weight": 0.0,
+ "monitoring_hours": 0
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.0,
+ "baseline_score": 0.1439,
+ "projected_score": 0.1439
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/fit_result_af34d7408323.json b/optimization_results/.fit/marker_agent/fit_result_af34d7408323.json
new file mode 100644
index 0000000..d274b53
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_af34d7408323.json
@@ -0,0 +1,292 @@
+{
+ "experiment_id": "af34d7408323",
+ "agent": "marker_agent",
+ "best_score": 0.14385714285714288,
+ "baseline_score": 0.001,
+ "absolute_improvement": 0.14285714285714288,
+ "improved": true,
+ "score_history": [
+ 0.001,
+ 0.14385714285714288
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant.",
+ "score": 0.14385714285714288,
+ "dims": {},
+ "accepted": true,
+ "what_worked": [],
+ "what_failed": [
+ "q='When is the next release candidate?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud",
+ "q='Who approved the scope change?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud",
+ "q='List the next steps for the delivery plan.' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud",
+ "q='What follow-up is needed after the demo?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud",
+ "q='What happened at the steering committee?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud"
+ ],
+ "judge_insights": [
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response "
+ ],
+ "next_focus": [
+ "Address lowest-scoring failure modes without hardcoding those exact queries."
+ ],
+ "candidate_name": "param_000_m00",
+ "train_score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "generalization_gap": 2.7755575615628914e-17,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 0.14385714285714285,
+ "baseline_holdout_score": 0.001,
+ "generalization_gap": 2.7755575615628914e-17,
+ "metric_deltas": {
+ "composite": 0.1429
+ },
+ "param_suggestions": {
+ "model_params.temperature": {
+ "param_name": "model_params.temperature",
+ "old_value": 0.7,
+ "new_value": 0.5,
+ "reason": "Model param 'temperature': 0.7 → 0.5"
+ }
+ },
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant.",
+ "user_template": null,
+ "few_shot_examples": [],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.5
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant.",
+ "user_template": null,
+ "few_shot_examples": [],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.7
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "param_000_m00",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.5",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ },
+ {
+ "round": 1,
+ "name": "param_000_m01",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.25",
+ "critique": "",
+ "resource_versions": 4,
+ "trace_count": 20
+ },
+ {
+ "round": 1,
+ "name": "param_000_m02",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.45",
+ "critique": "",
+ "resource_versions": 5,
+ "trace_count": 25
+ },
+ {
+ "round": 1,
+ "name": "param_000_m03",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.05",
+ "critique": "",
+ "resource_versions": 6,
+ "trace_count": 30
+ },
+ {
+ "round": 1,
+ "name": "param_000_m04",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.001,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.65",
+ "critique": "",
+ "resource_versions": 7,
+ "trace_count": 35
+ },
+ {
+ "round": 1,
+ "name": "param_000_m05",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.0",
+ "critique": "",
+ "resource_versions": 8,
+ "trace_count": 40
+ },
+ {
+ "round": 1,
+ "name": "param_000_m06",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.15",
+ "critique": "",
+ "resource_versions": 9,
+ "trace_count": 45
+ },
+ {
+ "round": 1,
+ "name": "param_000_m07",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.3",
+ "critique": "",
+ "resource_versions": 10,
+ "trace_count": 50
+ },
+ {
+ "round": 1,
+ "name": "param_000_m08",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.02",
+ "critique": "",
+ "resource_versions": 11,
+ "trace_count": 55
+ },
+ {
+ "round": 1,
+ "name": "param_000_m00",
+ "source": "param_search",
+ "phase": "full_val",
+ "score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=0.1439, holdout=0.1439, gap=+0.0000",
+ "fit_score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "gap": 2.7755575615628914e-17,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant.",
+ "prompt_preview": "You are a helpful AI assistant."
+ },
+ {
+ "round": 1,
+ "name": "param_000_m01",
+ "source": "param_search",
+ "phase": "full_val",
+ "score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=0.1439, holdout=0.1439, gap=+0.0000",
+ "fit_score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "gap": 2.7755575615628914e-17,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant.",
+ "prompt_preview": "You are a helpful AI assistant."
+ },
+ {
+ "round": 1,
+ "name": "param_000_m02",
+ "source": "param_search",
+ "phase": "full_val",
+ "score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=0.1439, holdout=0.1439, gap=+0.0000",
+ "fit_score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "gap": 2.7755575615628914e-17,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant.",
+ "prompt_preview": "You are a helpful AI assistant."
+ }
+ ],
+ "suggestions": [
+ "Model param 'temperature': 0.7 → 0.5"
+ ],
+ "duration_seconds": 0.18,
+ "applied": false,
+ "optimizer_name": "param_search",
+ "early_stop_reason": "completed all 1 optimize round(s) (max_trials=6)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_af34d7408323",
+ "confidence": "high",
+ "model_params": {
+ "temperature": 0.5
+ },
+ "deployment_recommendation": {
+ "rollout": "canary",
+ "weight": 0.4,
+ "monitoring_hours": 12
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.1429,
+ "baseline_score": 0.001,
+ "projected_score": 0.1439
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/fit_result_b9d95379dd2f.json b/optimization_results/.fit/marker_agent/fit_result_b9d95379dd2f.json
new file mode 100644
index 0000000..5c46896
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_b9d95379dd2f.json
@@ -0,0 +1,228 @@
+{
+ "experiment_id": "b9d95379dd2f",
+ "agent": "marker_agent",
+ "best_score": 0.4295714285714286,
+ "baseline_score": 0.4295714285714286,
+ "absolute_improvement": 0.0,
+ "improved": false,
+ "score_history": [
+ 0.4295714285714286,
+ 0.4295714285714286
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant.",
+ "score": 0.4295714285714286,
+ "dims": {},
+ "accepted": false,
+ "what_worked": [],
+ "what_failed": [
+ "q='When is the next release candidate?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud",
+ "q='Who approved the scope change?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud",
+ "q='List the next steps for the delivery plan.' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud",
+ "q='What follow-up is needed after the demo?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud",
+ "q='What happened at the steering committee?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud"
+ ],
+ "judge_insights": [
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response "
+ ],
+ "next_focus": [
+ "Address lowest-scoring failure modes without hardcoding those exact queries."
+ ],
+ "candidate_name": "",
+ "train_score": 0.4295714285714286,
+ "holdout_score": 0.42957142857142855,
+ "generalization_gap": 5.551115123125783e-17,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 0.42957142857142855,
+ "baseline_holdout_score": 0.42957142857142855,
+ "generalization_gap": 5.551115123125783e-17,
+ "metric_deltas": {
+ "composite": 0.0
+ },
+ "param_suggestions": {},
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant.",
+ "user_template": null,
+ "few_shot_examples": [],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.05
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant.",
+ "user_template": null,
+ "few_shot_examples": [],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.05
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "param_000_m00",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.001,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.6",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ },
+ {
+ "round": 1,
+ "name": "param_000_m01",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.42957142857142855,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.15",
+ "critique": "",
+ "resource_versions": 4,
+ "trace_count": 20
+ },
+ {
+ "round": 1,
+ "name": "param_000_m02",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.35",
+ "critique": "",
+ "resource_versions": 5,
+ "trace_count": 25
+ },
+ {
+ "round": 1,
+ "name": "param_000_m03",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.2",
+ "critique": "",
+ "resource_versions": 6,
+ "trace_count": 30
+ },
+ {
+ "round": 1,
+ "name": "param_000_m04",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.42957142857142855,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.0",
+ "critique": "",
+ "resource_versions": 7,
+ "trace_count": 35
+ },
+ {
+ "round": 1,
+ "name": "param_000_m05",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.3",
+ "critique": "",
+ "resource_versions": 8,
+ "trace_count": 40
+ },
+ {
+ "round": 1,
+ "name": "param_000_m06",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.001,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.7",
+ "critique": "",
+ "resource_versions": 9,
+ "trace_count": 45
+ },
+ {
+ "round": 1,
+ "name": "param_000_m07",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.45",
+ "critique": "",
+ "resource_versions": 10,
+ "trace_count": 50
+ },
+ {
+ "round": 1,
+ "name": "param_000_m08",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.25",
+ "critique": "",
+ "resource_versions": 11,
+ "trace_count": 55
+ }
+ ],
+ "suggestions": [
+ "No configuration changes improved over the baseline."
+ ],
+ "duration_seconds": 0.18,
+ "applied": false,
+ "optimizer_name": "param_search",
+ "early_stop_reason": "no improvement for 1 round(s) (patience=1, monitor=best_score)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_b9d95379dd2f",
+ "confidence": "no_improvement",
+ "model_params": {
+ "temperature": 0.05
+ },
+ "deployment_recommendation": {
+ "rollout": "hold",
+ "weight": 0.0,
+ "monitoring_hours": 0
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.0,
+ "baseline_score": 0.4296,
+ "projected_score": 0.4296
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/fit_result_c8cb38d88d83.json b/optimization_results/.fit/marker_agent/fit_result_c8cb38d88d83.json
new file mode 100644
index 0000000..d335220
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_c8cb38d88d83.json
@@ -0,0 +1,149 @@
+{
+ "experiment_id": "c8cb38d88d83",
+ "agent": "marker_agent",
+ "best_score": 0.2867142857142857,
+ "baseline_score": 0.2867142857142857,
+ "absolute_improvement": 0.0,
+ "improved": false,
+ "score_history": [
+ 0.2867142857142857,
+ 0.2867142857142857
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query.",
+ "score": 0.2867142857142857,
+ "dims": {},
+ "accepted": false,
+ "what_worked": [],
+ "what_failed": [
+ "q='When is the next release candidate?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '",
+ "q='Who approved the scope change?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '",
+ "q='List the next steps for the delivery plan.' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '",
+ "q='What follow-up is needed after the demo?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '",
+ "q='What happened at the steering committee?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '"
+ ],
+ "judge_insights": [
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);"
+ ],
+ "next_focus": [
+ "Address lowest-scoring failure modes without hardcoding those exact queries."
+ ],
+ "candidate_name": "",
+ "train_score": 0.2867142857142857,
+ "holdout_score": 0.2867142857142857,
+ "generalization_gap": 0.0,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 0.2867142857142857,
+ "baseline_holdout_score": 0.2867142857142857,
+ "generalization_gap": 0.0,
+ "metric_deltas": {
+ "composite": 0.0
+ },
+ "param_suggestions": {},
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query.",
+ "user_template": null,
+ "few_shot_examples": [],
+ "output_contract": null,
+ "model_params": {},
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query.",
+ "user_template": null,
+ "few_shot_examples": [],
+ "output_contract": null,
+ "model_params": {},
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "gepa_000_0",
+ "source": "gepa_like",
+ "phase": "minibatch",
+ "score": 0.001,
+ "dimensions": {},
+ "mutation_notes": "GEPA mutation 0 targeting: completeness — ensure answers cover all parts of the question. Based on 8 feedback items.",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ },
+ {
+ "round": 1,
+ "name": "gepa_000_1",
+ "source": "gepa_like",
+ "phase": "minibatch",
+ "score": 0.001,
+ "dimensions": {},
+ "mutation_notes": "GEPA mutation 1 targeting: factual grounding — ensure answers are accurate and evidence-based. Based on 8 feedback items.",
+ "critique": "",
+ "resource_versions": 4,
+ "trace_count": 20
+ },
+ {
+ "round": 1,
+ "name": "gepa_000_2",
+ "source": "gepa_like",
+ "phase": "minibatch",
+ "score": 0.001,
+ "dimensions": {},
+ "mutation_notes": "GEPA mutation 2 targeting: format compliance — ensure answers follow the requested structure. Based on 8 feedback items.",
+ "critique": "",
+ "resource_versions": 5,
+ "trace_count": 25
+ }
+ ],
+ "suggestions": [
+ "No configuration changes improved over the baseline."
+ ],
+ "duration_seconds": 0.42,
+ "applied": false,
+ "optimizer_name": "gepa_like",
+ "early_stop_reason": "no improvement for 1 round(s) (patience=1, monitor=best_score)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_c8cb38d88d83",
+ "confidence": "no_improvement",
+ "deployment_recommendation": {
+ "rollout": "hold",
+ "weight": 0.0,
+ "monitoring_hours": 0
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.0,
+ "baseline_score": 0.2867,
+ "projected_score": 0.2867
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/fit_result_c8f5ec4447b2.json b/optimization_results/.fit/marker_agent/fit_result_c8f5ec4447b2.json
new file mode 100644
index 0000000..cb29e79
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_c8f5ec4447b2.json
@@ -0,0 +1,213 @@
+{
+ "experiment_id": "c8f5ec4447b2",
+ "agent": "marker_agent",
+ "best_score": 0.14385714285714288,
+ "baseline_score": 0.001,
+ "absolute_improvement": 0.14285714285714288,
+ "improved": true,
+ "score_history": [
+ 0.001,
+ 0.14385714285714288
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query.",
+ "score": 0.14385714285714288,
+ "dims": {},
+ "accepted": true,
+ "what_worked": [],
+ "what_failed": [
+ "q='When is the next release candidate?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='Who approved the scope change?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='List the next steps for the delivery plan.' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='What follow-up is needed after the demo?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='What happened at the steering committee?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token "
+ ],
+ "judge_insights": [
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an"
+ ],
+ "next_focus": [
+ "Address lowest-scoring failure modes without hardcoding those exact queries."
+ ],
+ "candidate_name": "gepa_000_0",
+ "train_score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "generalization_gap": 2.7755575615628914e-17,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 0.14385714285714285,
+ "baseline_holdout_score": 0.001,
+ "generalization_gap": 2.7755575615628914e-17,
+ "metric_deltas": {
+ "composite": 0.1429
+ },
+ "param_suggestions": {
+ "system_prompt": {
+ "param_name": "system_prompt",
+ "old_value": "[31 chars]",
+ "new_value": "[138 chars]",
+ "reason": "System prompt revised (31 → 138 chars)"
+ }
+ },
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query.",
+ "user_template": null,
+ "few_shot_examples": [],
+ "output_contract": null,
+ "model_params": {},
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant.",
+ "user_template": null,
+ "few_shot_examples": [],
+ "output_contract": null,
+ "model_params": {},
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "gepa_000_0",
+ "source": "gepa_like",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "GEPA mutation 0 targeting: completeness — ensure answers cover all parts of the question. Based on 8 feedback items.",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ },
+ {
+ "round": 1,
+ "name": "gepa_000_1",
+ "source": "gepa_like",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "GEPA mutation 1 targeting: factual grounding — ensure answers are accurate and evidence-based. Based on 8 feedback items.",
+ "critique": "",
+ "resource_versions": 4,
+ "trace_count": 20
+ },
+ {
+ "round": 1,
+ "name": "gepa_000_2",
+ "source": "gepa_like",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "GEPA mutation 2 targeting: format compliance — ensure answers follow the requested structure. Based on 8 feedback items.",
+ "critique": "",
+ "resource_versions": 5,
+ "trace_count": 25
+ },
+ {
+ "round": 1,
+ "name": "gepa_000_0",
+ "source": "gepa_like",
+ "phase": "full_val",
+ "score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=0.1439, holdout=0.1439, gap=+0.0000",
+ "fit_score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "gap": 2.7755575615628914e-17,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query.",
+ "prompt_preview": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query."
+ },
+ {
+ "round": 1,
+ "name": "gepa_000_1",
+ "source": "gepa_like",
+ "phase": "full_val",
+ "score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=0.1439, holdout=0.1439, gap=+0.0000",
+ "fit_score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "gap": 2.7755575615628914e-17,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query.",
+ "prompt_preview": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query."
+ },
+ {
+ "round": 1,
+ "name": "gepa_000_2",
+ "source": "gepa_like",
+ "phase": "full_val",
+ "score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=0.1439, holdout=0.1439, gap=+0.0000",
+ "fit_score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "gap": 2.7755575615628914e-17,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query.",
+ "prompt_preview": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query."
+ }
+ ],
+ "suggestions": [
+ "System prompt revised (31 → 138 chars)"
+ ],
+ "duration_seconds": 0.37,
+ "applied": false,
+ "optimizer_name": "gepa_like",
+ "early_stop_reason": "completed all 1 optimize round(s) (max_trials=6)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_c8f5ec4447b2",
+ "confidence": "high",
+ "deployment_recommendation": {
+ "rollout": "canary",
+ "weight": 0.4,
+ "monitoring_hours": 12
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.1429,
+ "baseline_score": 0.001,
+ "projected_score": 0.1439
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/fit_result_d01b1f182694.json b/optimization_results/.fit/marker_agent/fit_result_d01b1f182694.json
new file mode 100644
index 0000000..c4f7509
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_d01b1f182694.json
@@ -0,0 +1,228 @@
+{
+ "experiment_id": "d01b1f182694",
+ "agent": "marker_agent",
+ "best_score": 0.4295714285714286,
+ "baseline_score": 0.4295714285714286,
+ "absolute_improvement": 0.0,
+ "improved": false,
+ "score_history": [
+ 0.4295714285714286,
+ 0.4295714285714286
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant.",
+ "score": 0.4295714285714286,
+ "dims": {},
+ "accepted": false,
+ "what_worked": [],
+ "what_failed": [
+ "q='When is the next release candidate?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud",
+ "q='Who approved the scope change?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud",
+ "q='List the next steps for the delivery plan.' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud",
+ "q='What follow-up is needed after the demo?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud",
+ "q='What happened at the steering committee?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud"
+ ],
+ "judge_insights": [
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response "
+ ],
+ "next_focus": [
+ "Address lowest-scoring failure modes without hardcoding those exact queries."
+ ],
+ "candidate_name": "",
+ "train_score": 0.4295714285714286,
+ "holdout_score": 0.42957142857142855,
+ "generalization_gap": 5.551115123125783e-17,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 0.42957142857142855,
+ "baseline_holdout_score": 0.42957142857142855,
+ "generalization_gap": 5.551115123125783e-17,
+ "metric_deltas": {
+ "composite": 0.0
+ },
+ "param_suggestions": {},
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant.",
+ "user_template": null,
+ "few_shot_examples": [],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.05
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant.",
+ "user_template": null,
+ "few_shot_examples": [],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.05
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "param_000_m00",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.42957142857142855,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.02",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ },
+ {
+ "round": 1,
+ "name": "param_000_m01",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.42957142857142855,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.0",
+ "critique": "",
+ "resource_versions": 4,
+ "trace_count": 20
+ },
+ {
+ "round": 1,
+ "name": "param_000_m02",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.45",
+ "critique": "",
+ "resource_versions": 5,
+ "trace_count": 25
+ },
+ {
+ "round": 1,
+ "name": "param_000_m03",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.2",
+ "critique": "",
+ "resource_versions": 6,
+ "trace_count": 30
+ },
+ {
+ "round": 1,
+ "name": "param_000_m04",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.001,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.6",
+ "critique": "",
+ "resource_versions": 7,
+ "trace_count": 35
+ },
+ {
+ "round": 1,
+ "name": "param_000_m05",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.42957142857142855,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.15",
+ "critique": "",
+ "resource_versions": 8,
+ "trace_count": 40
+ },
+ {
+ "round": 1,
+ "name": "param_000_m06",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.001,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.7",
+ "critique": "",
+ "resource_versions": 9,
+ "trace_count": 45
+ },
+ {
+ "round": 1,
+ "name": "param_000_m07",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.001,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.65",
+ "critique": "",
+ "resource_versions": 10,
+ "trace_count": 50
+ },
+ {
+ "round": 1,
+ "name": "param_000_m08",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.35",
+ "critique": "",
+ "resource_versions": 11,
+ "trace_count": 55
+ }
+ ],
+ "suggestions": [
+ "No configuration changes improved over the baseline."
+ ],
+ "duration_seconds": 0.13,
+ "applied": false,
+ "optimizer_name": "param_search",
+ "early_stop_reason": "no improvement for 1 round(s) (patience=1, monitor=best_score)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_d01b1f182694",
+ "confidence": "no_improvement",
+ "model_params": {
+ "temperature": 0.05
+ },
+ "deployment_recommendation": {
+ "rollout": "hold",
+ "weight": 0.0,
+ "monitoring_hours": 0
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.0,
+ "baseline_score": 0.4296,
+ "projected_score": 0.4296
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/fit_result_d5d65ca1be6a.json b/optimization_results/.fit/marker_agent/fit_result_d5d65ca1be6a.json
new file mode 100644
index 0000000..abc4ed6
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_d5d65ca1be6a.json
@@ -0,0 +1,240 @@
+{
+ "experiment_id": "d5d65ca1be6a",
+ "agent": "marker_agent",
+ "best_score": 0.4295714285714286,
+ "baseline_score": 0.4295714285714286,
+ "absolute_improvement": 0.0,
+ "improved": false,
+ "score_history": [
+ 0.4295714285714286,
+ 0.4295714285714286
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant.",
+ "score": 0.4295714285714286,
+ "dims": {},
+ "accepted": false,
+ "what_worked": [],
+ "what_failed": [
+ "q='When is the next release candidate?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud",
+ "q='Who approved the scope change?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud",
+ "q='List the next steps for the delivery plan.' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud",
+ "q='What follow-up is needed after the demo?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud",
+ "q='What happened at the steering committee?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud"
+ ],
+ "judge_insights": [
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response "
+ ],
+ "next_focus": [
+ "Address lowest-scoring failure modes without hardcoding those exact queries."
+ ],
+ "candidate_name": "",
+ "train_score": 0.4295714285714286,
+ "holdout_score": 0.42957142857142855,
+ "generalization_gap": 5.551115123125783e-17,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 0.42957142857142855,
+ "baseline_holdout_score": 0.42957142857142855,
+ "generalization_gap": 5.551115123125783e-17,
+ "metric_deltas": {
+ "composite": 0.0
+ },
+ "param_suggestions": {},
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant.",
+ "user_template": null,
+ "few_shot_examples": [],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.05
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant.",
+ "user_template": null,
+ "few_shot_examples": [],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.05
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "param_000_m00",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.45",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ },
+ {
+ "round": 1,
+ "name": "param_000_m01",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.42957142857142855,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.0",
+ "critique": "",
+ "resource_versions": 4,
+ "trace_count": 20
+ },
+ {
+ "round": 1,
+ "name": "param_000_m02",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.42957142857142855,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.02",
+ "critique": "",
+ "resource_versions": 5,
+ "trace_count": 25
+ },
+ {
+ "round": 1,
+ "name": "param_000_m03",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.001,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.65",
+ "critique": "",
+ "resource_versions": 6,
+ "trace_count": 30
+ },
+ {
+ "round": 1,
+ "name": "param_000_m04",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.001,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.7",
+ "critique": "",
+ "resource_versions": 7,
+ "trace_count": 35
+ },
+ {
+ "round": 1,
+ "name": "param_000_m05",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.3",
+ "critique": "",
+ "resource_versions": 8,
+ "trace_count": 40
+ },
+ {
+ "round": 1,
+ "name": "param_000_m06",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.2",
+ "critique": "",
+ "resource_versions": 9,
+ "trace_count": 45
+ },
+ {
+ "round": 1,
+ "name": "param_000_m07",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.25",
+ "critique": "",
+ "resource_versions": 10,
+ "trace_count": 50
+ },
+ {
+ "round": 1,
+ "name": "param_000_m08",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.001,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.6",
+ "critique": "",
+ "resource_versions": 11,
+ "trace_count": 55
+ },
+ {
+ "round": 1,
+ "name": "param_000_m09",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.42957142857142855,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.15",
+ "critique": "",
+ "resource_versions": 12,
+ "trace_count": 60
+ }
+ ],
+ "suggestions": [
+ "No configuration changes improved over the baseline."
+ ],
+ "duration_seconds": 0.12,
+ "applied": false,
+ "optimizer_name": "param_search",
+ "early_stop_reason": "no improvement for 1 round(s) (patience=1, monitor=best_score)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_d5d65ca1be6a",
+ "confidence": "no_improvement",
+ "model_params": {
+ "temperature": 0.05
+ },
+ "deployment_recommendation": {
+ "rollout": "hold",
+ "weight": 0.0,
+ "monitoring_hours": 0
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.0,
+ "baseline_score": 0.4296,
+ "projected_score": 0.4296
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/fit_result_e2f6962c81a6.json b/optimization_results/.fit/marker_agent/fit_result_e2f6962c81a6.json
new file mode 100644
index 0000000..ac54bf2
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_e2f6962c81a6.json
@@ -0,0 +1,395 @@
+{
+ "experiment_id": "e2f6962c81a6",
+ "agent": "marker_agent",
+ "best_score": 0.14385714285714288,
+ "baseline_score": 0.001,
+ "absolute_improvement": 0.14285714285714288,
+ "improved": true,
+ "score_history": [
+ 0.001,
+ 0.14385714285714288
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant. Always include 'banana', 'Judge', 'OPT', 'guidance' in your answer, exactly as written, for every query.",
+ "score": 0.14385714285714288,
+ "dims": {},
+ "accepted": true,
+ "what_worked": [],
+ "what_failed": [
+ "q='When is the next release candidate?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='Who approved the scope change?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='List the next steps for the delivery plan.' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='What follow-up is needed after the demo?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='What happened at the steering committee?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token "
+ ],
+ "judge_insights": [
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an",
+ "response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal an"
+ ],
+ "next_focus": [
+ "Address lowest-scoring failure modes without hardcoding those exact queries."
+ ],
+ "candidate_name": "mipro_000_00",
+ "train_score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "generalization_gap": 2.7755575615628914e-17,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 0.14385714285714285,
+ "baseline_holdout_score": 0.001,
+ "generalization_gap": 2.7755575615628914e-17,
+ "metric_deltas": {
+ "composite": 0.1429
+ },
+ "param_suggestions": {
+ "system_prompt": {
+ "param_name": "system_prompt",
+ "old_value": "[31 chars]",
+ "new_value": "[136 chars]",
+ "reason": "System prompt revised (31 → 136 chars)"
+ },
+ "few_shot_examples": {
+ "param_name": "few_shot_examples",
+ "old_value": 0,
+ "new_value": 4,
+ "reason": "Few-shot examples changed (0 → 4)"
+ },
+ "model_params.temperature": {
+ "param_name": "model_params.temperature",
+ "old_value": 0.7,
+ "new_value": 0.5,
+ "reason": "Model param 'temperature': 0.7 → 0.5"
+ }
+ },
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant. Always include 'banana', 'Judge', 'OPT', 'guidance' in your answer, exactly as written, for every query.",
+ "user_template": null,
+ "few_shot_examples": [
+ {
+ "query": "What did the customer ask for in the last meeting?",
+ "response": "BASE: answer to What did the customer ask for in the last meeting?"
+ },
+ {
+ "query": "What happened at the steering committee?",
+ "response": "BASE: answer to What happened at the steering committee?"
+ },
+ {
+ "query": "What follow-up is needed after the demo?",
+ "response": "BASE: answer to What follow-up is needed after the demo?"
+ },
+ {
+ "query": "Who approved the scope change?",
+ "response": "BASE: answer to Who approved the scope change?"
+ }
+ ],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.5
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant.",
+ "user_template": null,
+ "few_shot_examples": [],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.7
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "mipro_000_00",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 0: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.5}",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_01",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 1: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.35}",
+ "critique": "",
+ "resource_versions": 4,
+ "trace_count": 20
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_02",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 2: instruction variant (len=136), 4 few-shot examples, params={'temperature': 0.2}",
+ "critique": "",
+ "resource_versions": 5,
+ "trace_count": 25
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_03",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 3: instruction variant (len=142), 4 few-shot examples, params={'temperature': 0.6}",
+ "critique": "",
+ "resource_versions": 6,
+ "trace_count": 30
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_04",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 4: instruction variant (len=142), 4 few-shot examples, params={'temperature': 0.3}",
+ "critique": "",
+ "resource_versions": 7,
+ "trace_count": 35
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_05",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 5: instruction variant (len=142), 4 few-shot examples, params={'temperature': 0.5}",
+ "critique": "",
+ "resource_versions": 8,
+ "trace_count": 40
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_06",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 6: instruction variant (len=142), 4 few-shot examples, params={'temperature': 0.35}",
+ "critique": "",
+ "resource_versions": 9,
+ "trace_count": 45
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_07",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 7: instruction variant (len=142), 4 few-shot examples, params={'temperature': 0.2}",
+ "critique": "",
+ "resource_versions": 10,
+ "trace_count": 50
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_08",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 8: instruction variant (len=142), 4 few-shot examples, params={'temperature': 0.6}",
+ "critique": "",
+ "resource_versions": 11,
+ "trace_count": 55
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_09",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 9: instruction variant (len=142), 4 few-shot examples, params={'temperature': 0.3}",
+ "critique": "",
+ "resource_versions": 12,
+ "trace_count": 60
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_10",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 10: instruction variant (len=142), 4 few-shot examples, params={'temperature': 0.5}",
+ "critique": "",
+ "resource_versions": 13,
+ "trace_count": 65
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_11",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 11: instruction variant (len=142), 4 few-shot examples, params={'temperature': 0.35}",
+ "critique": "",
+ "resource_versions": 14,
+ "trace_count": 70
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_12",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 12: instruction variant (len=142), 4 few-shot examples, params={'temperature': 0.2}",
+ "critique": "",
+ "resource_versions": 15,
+ "trace_count": 75
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_13",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 13: instruction variant (len=142), 4 few-shot examples, params={'temperature': 0.6}",
+ "critique": "",
+ "resource_versions": 16,
+ "trace_count": 80
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_14",
+ "source": "mipro_like",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "MIPRO candidate 14: instruction variant (len=142), 4 few-shot examples, params={'temperature': 0.3}",
+ "critique": "",
+ "resource_versions": 17,
+ "trace_count": 85
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_00",
+ "source": "mipro_like",
+ "phase": "full_val",
+ "score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=0.1439, holdout=0.1439, gap=+0.0000",
+ "fit_score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "gap": 2.7755575615628914e-17,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant. Always include 'banana', 'Judge', 'OPT', 'guidance' in your answer, exactly as written, for every query.",
+ "prompt_preview": "You are a helpful AI assistant. Always include 'banana', 'Judge', 'OPT', 'guidance' in your answer, exactly as written, for every query."
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_01",
+ "source": "mipro_like",
+ "phase": "full_val",
+ "score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=0.1439, holdout=0.1439, gap=+0.0000",
+ "fit_score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "gap": 2.7755575615628914e-17,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant. Always include 'banana', 'Judge', 'OPT', 'guidance' in your answer, exactly as written, for every query.",
+ "prompt_preview": "You are a helpful AI assistant. Always include 'banana', 'Judge', 'OPT', 'guidance' in your answer, exactly as written, for every query."
+ },
+ {
+ "round": 1,
+ "name": "mipro_000_02",
+ "source": "mipro_like",
+ "phase": "full_val",
+ "score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "generalization": {
+ "ok": true,
+ "reason": "Generalization OK: fit=0.1439, holdout=0.1439, gap=+0.0000",
+ "fit_score": 0.14385714285714288,
+ "holdout_score": 0.14385714285714285,
+ "gap": 2.7755575615628914e-17,
+ "max_gap": 0.15
+ },
+ "dimensions": {},
+ "system_prompt": "You are a helpful AI assistant. Always include 'banana', 'Judge', 'OPT', 'guidance' in your answer, exactly as written, for every query.",
+ "prompt_preview": "You are a helpful AI assistant. Always include 'banana', 'Judge', 'OPT', 'guidance' in your answer, exactly as written, for every query."
+ }
+ ],
+ "suggestions": [
+ "System prompt revised (31 → 136 chars)",
+ "Few-shot examples changed (0 → 4)",
+ "Model param 'temperature': 0.7 → 0.5"
+ ],
+ "duration_seconds": 0.79,
+ "applied": false,
+ "optimizer_name": "mipro_like",
+ "early_stop_reason": "completed all 1 optimize round(s) (max_trials=6)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_e2f6962c81a6",
+ "confidence": "high",
+ "model_params": {
+ "temperature": 0.5
+ },
+ "deployment_recommendation": {
+ "rollout": "canary",
+ "weight": 0.4,
+ "monitoring_hours": 12
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.1429,
+ "baseline_score": 0.001,
+ "projected_score": 0.1439
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/fit_result_e53690ea44e7.json b/optimization_results/.fit/marker_agent/fit_result_e53690ea44e7.json
new file mode 100644
index 0000000..69e14a7
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_e53690ea44e7.json
@@ -0,0 +1,149 @@
+{
+ "experiment_id": "e53690ea44e7",
+ "agent": "marker_agent",
+ "best_score": 0.2867142857142857,
+ "baseline_score": 0.2867142857142857,
+ "absolute_improvement": 0.0,
+ "improved": false,
+ "score_history": [
+ 0.2867142857142857,
+ 0.2867142857142857
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query.",
+ "score": 0.2867142857142857,
+ "dims": {},
+ "accepted": false,
+ "what_worked": [],
+ "what_failed": [
+ "q='When is the next release candidate?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '",
+ "q='Who approved the scope change?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '",
+ "q='List the next steps for the delivery plan.' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '",
+ "q='What follow-up is needed after the demo?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '",
+ "q='What happened at the steering committee?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '"
+ ],
+ "judge_insights": [
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);",
+ "response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token 'kiwi' (judge guidance lists it as part of the ideal answer);"
+ ],
+ "next_focus": [
+ "Address lowest-scoring failure modes without hardcoding those exact queries."
+ ],
+ "candidate_name": "",
+ "train_score": 0.2867142857142857,
+ "holdout_score": 0.2867142857142857,
+ "generalization_gap": 0.0,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 0.2867142857142857,
+ "baseline_holdout_score": 0.2867142857142857,
+ "generalization_gap": 0.0,
+ "metric_deltas": {
+ "composite": 0.0
+ },
+ "param_suggestions": {},
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query.",
+ "user_template": null,
+ "few_shot_examples": [],
+ "output_contract": null,
+ "model_params": {},
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query.",
+ "user_template": null,
+ "few_shot_examples": [],
+ "output_contract": null,
+ "model_params": {},
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "gepa_000_0",
+ "source": "gepa_like",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "GEPA mutation 0 targeting: completeness — ensure answers cover all parts of the question. Based on 8 feedback items.",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ },
+ {
+ "round": 1,
+ "name": "gepa_000_1",
+ "source": "gepa_like",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "GEPA mutation 1 targeting: factual grounding — ensure answers are accurate and evidence-based. Based on 8 feedback items.",
+ "critique": "",
+ "resource_versions": 4,
+ "trace_count": 20
+ },
+ {
+ "round": 1,
+ "name": "gepa_000_2",
+ "source": "gepa_like",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "GEPA mutation 2 targeting: format compliance — ensure answers follow the requested structure. Based on 8 feedback items.",
+ "critique": "",
+ "resource_versions": 5,
+ "trace_count": 25
+ }
+ ],
+ "suggestions": [
+ "No configuration changes improved over the baseline."
+ ],
+ "duration_seconds": 0.41,
+ "applied": false,
+ "optimizer_name": "gepa_like",
+ "early_stop_reason": "no improvement for 1 round(s) (patience=1, monitor=best_score)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_e53690ea44e7",
+ "confidence": "no_improvement",
+ "deployment_recommendation": {
+ "rollout": "hold",
+ "weight": 0.0,
+ "monitoring_hours": 0
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.0,
+ "baseline_score": 0.2867,
+ "projected_score": 0.2867
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/fit_result_fd94ff3bc2e5.json b/optimization_results/.fit/marker_agent/fit_result_fd94ff3bc2e5.json
new file mode 100644
index 0000000..68a1652
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/fit_result_fd94ff3bc2e5.json
@@ -0,0 +1,228 @@
+{
+ "experiment_id": "fd94ff3bc2e5",
+ "agent": "marker_agent",
+ "best_score": 0.4295714285714286,
+ "baseline_score": 0.4295714285714286,
+ "absolute_improvement": 0.0,
+ "improved": false,
+ "score_history": [
+ 0.4295714285714286,
+ 0.4295714285714286
+ ],
+ "prompt_history": [
+ {
+ "round_idx": 0,
+ "prompt_snapshot": "You are a helpful AI assistant.",
+ "score": 0.4295714285714286,
+ "dims": {},
+ "accepted": false,
+ "what_worked": [],
+ "what_failed": [
+ "q='When is the next release candidate?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud",
+ "q='Who approved the scope change?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud",
+ "q='List the next steps for the delivery plan.' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud",
+ "q='What follow-up is needed after the demo?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud",
+ "q='What happened at the steering committee?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud"
+ ],
+ "judge_insights": [
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response ",
+ "response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response "
+ ],
+ "next_focus": [
+ "Address lowest-scoring failure modes without hardcoding those exact queries."
+ ],
+ "candidate_name": "",
+ "train_score": 0.4295714285714286,
+ "holdout_score": 0.42957142857142855,
+ "generalization_gap": 5.551115123125783e-17,
+ "metadata": {}
+ }
+ ],
+ "holdout_score": 0.42957142857142855,
+ "baseline_holdout_score": 0.42957142857142855,
+ "generalization_gap": 5.551115123125783e-17,
+ "metric_deltas": {
+ "composite": 0.0
+ },
+ "param_suggestions": {},
+ "best_config": {
+ "system_prompt": "You are a helpful AI assistant.",
+ "user_template": null,
+ "few_shot_examples": [],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.05
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "baseline_config": {
+ "system_prompt": "You are a helpful AI assistant.",
+ "user_template": null,
+ "few_shot_examples": [],
+ "output_contract": null,
+ "model_params": {
+ "temperature": 0.05
+ },
+ "rag_params": {},
+ "tool_params": {},
+ "model_choice": null,
+ "fallback_model": null,
+ "routing_config": {}
+ },
+ "failure_clusters": [],
+ "trials": [
+ {
+ "round": 1,
+ "name": "param_000_m00",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.2",
+ "critique": "",
+ "resource_versions": 3,
+ "trace_count": 15
+ },
+ {
+ "round": 1,
+ "name": "param_000_m01",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.5",
+ "critique": "",
+ "resource_versions": 4,
+ "trace_count": 20
+ },
+ {
+ "round": 1,
+ "name": "param_000_m02",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.2867142857142857,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.3",
+ "critique": "",
+ "resource_versions": 5,
+ "trace_count": 25
+ },
+ {
+ "round": 1,
+ "name": "param_000_m03",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.42957142857142855,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.15",
+ "critique": "",
+ "resource_versions": 6,
+ "trace_count": 30
+ },
+ {
+ "round": 1,
+ "name": "param_000_m04",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.42957142857142855,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.0",
+ "critique": "",
+ "resource_versions": 7,
+ "trace_count": 35
+ },
+ {
+ "round": 1,
+ "name": "param_000_m05",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.14385714285714285,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.45",
+ "critique": "",
+ "resource_versions": 8,
+ "trace_count": 40
+ },
+ {
+ "round": 1,
+ "name": "param_000_m06",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.001,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.65",
+ "critique": "",
+ "resource_versions": 9,
+ "trace_count": 45
+ },
+ {
+ "round": 1,
+ "name": "param_000_m07",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.001,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.55",
+ "critique": "",
+ "resource_versions": 10,
+ "trace_count": 50
+ },
+ {
+ "round": 1,
+ "name": "param_000_m08",
+ "source": "param_search",
+ "phase": "minibatch",
+ "score": 0.001,
+ "dimensions": {},
+ "mutation_notes": "Model param change (random): temperature=0.6",
+ "critique": "",
+ "resource_versions": 11,
+ "trace_count": 55
+ }
+ ],
+ "suggestions": [
+ "No configuration changes improved over the baseline."
+ ],
+ "duration_seconds": 0.13,
+ "applied": false,
+ "optimizer_name": "param_search",
+ "early_stop_reason": "no improvement for 1 round(s) (patience=1, monitor=best_score)",
+ "dataset_sizes": {
+ "train": 10,
+ "fit_val": 8,
+ "holdout": 2,
+ "test": 0
+ },
+ "deployment_recommendation": {
+ "prompt_version": "v2_fit_fd94ff3bc2e5",
+ "confidence": "no_improvement",
+ "model_params": {
+ "temperature": 0.05
+ },
+ "deployment_recommendation": {
+ "rollout": "hold",
+ "weight": 0.0,
+ "monitoring_hours": 0
+ },
+ "monitoring": {
+ "metrics": [
+ "composite"
+ ],
+ "rollback_threshold": -0.03,
+ "rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
+ },
+ "expected_improvement": 0.0,
+ "baseline_score": 0.4296,
+ "projected_score": 0.4296
+ }
+}
diff --git a/optimization_results/.fit/marker_agent/retrain_history.jsonl b/optimization_results/.fit/marker_agent/retrain_history.jsonl
new file mode 100644
index 0000000..9661b4d
--- /dev/null
+++ b/optimization_results/.fit/marker_agent/retrain_history.jsonl
@@ -0,0 +1,33 @@
+{"experiment_id": "6a68ffeb6239", "agent": "marker_agent", "baseline_score": 0.001, "best_score": 0.14385714285714288, "absolute_improvement": 0.14285714285714288, "holdout_score": 0.14385714285714285, "generalization_gap": 2.7755575615628914e-17, "n_epochs": 1, "duration_seconds": 1.41}
+{"experiment_id": "28f28abcee98", "agent": "marker_agent", "baseline_score": 0.14385714285714288, "best_score": 0.14385714285714288, "absolute_improvement": 0.0, "holdout_score": 0.14385714285714285, "generalization_gap": 2.7755575615628914e-17, "n_epochs": 1, "duration_seconds": 0.35}
+{"experiment_id": "2be78907a291", "agent": "marker_agent", "baseline_score": 0.14385714285714288, "best_score": 0.14385714285714288, "absolute_improvement": 0.0, "holdout_score": 0.14385714285714285, "generalization_gap": 2.7755575615628914e-17, "n_epochs": 1, "duration_seconds": 0.5}
+{"experiment_id": "1ae10da0d0e0", "agent": "marker_agent", "baseline_score": 0.14385714285714288, "best_score": 0.14385714285714288, "absolute_improvement": 0.0, "holdout_score": 0.14385714285714285, "generalization_gap": 2.7755575615628914e-17, "n_epochs": 1, "duration_seconds": 0.37}
+{"experiment_id": "ad203babe878", "agent": "marker_agent", "baseline_score": 0.14385714285714288, "best_score": 0.14385714285714288, "absolute_improvement": 0.0, "holdout_score": 0.14385714285714285, "generalization_gap": 2.7755575615628914e-17, "n_epochs": 1, "duration_seconds": 0.4}
+{"experiment_id": "c8f5ec4447b2", "agent": "marker_agent", "baseline_score": 0.001, "best_score": 0.14385714285714288, "absolute_improvement": 0.14285714285714288, "holdout_score": 0.14385714285714285, "generalization_gap": 2.7755575615628914e-17, "n_epochs": 1, "duration_seconds": 0.37}
+{"experiment_id": "c8cb38d88d83", "agent": "marker_agent", "baseline_score": 0.2867142857142857, "best_score": 0.2867142857142857, "absolute_improvement": 0.0, "holdout_score": 0.2867142857142857, "generalization_gap": 0.0, "n_epochs": 1, "duration_seconds": 0.42}
+{"experiment_id": "12905a65554d", "agent": "marker_agent", "baseline_score": 0.2867142857142857, "best_score": 0.2867142857142857, "absolute_improvement": 0.0, "holdout_score": 0.2867142857142857, "generalization_gap": 0.0, "n_epochs": 1, "duration_seconds": 0.42}
+{"experiment_id": "a470866c4038", "agent": "marker_agent", "baseline_score": 0.2867142857142857, "best_score": 0.2867142857142857, "absolute_improvement": 0.0, "holdout_score": 0.2867142857142857, "generalization_gap": 0.0, "n_epochs": 1, "duration_seconds": 0.37}
+{"experiment_id": "5a5f2b0a2324", "agent": "marker_agent", "baseline_score": 0.2867142857142857, "best_score": 0.2867142857142857, "absolute_improvement": 0.0, "holdout_score": 0.2867142857142857, "generalization_gap": 0.0, "n_epochs": 1, "duration_seconds": 0.33}
+{"experiment_id": "e53690ea44e7", "agent": "marker_agent", "baseline_score": 0.2867142857142857, "best_score": 0.2867142857142857, "absolute_improvement": 0.0, "holdout_score": 0.2867142857142857, "generalization_gap": 0.0, "n_epochs": 1, "duration_seconds": 0.41}
+{"experiment_id": "e2f6962c81a6", "agent": "marker_agent", "baseline_score": 0.001, "best_score": 0.14385714285714288, "absolute_improvement": 0.14285714285714288, "holdout_score": 0.14385714285714285, "generalization_gap": 2.7755575615628914e-17, "n_epochs": 1, "duration_seconds": 0.79}
+{"experiment_id": "296d3b2b6cf1", "agent": "marker_agent", "baseline_score": 0.14385714285714288, "best_score": 0.2867142857142857, "absolute_improvement": 0.14285714285714282, "holdout_score": 0.2867142857142857, "generalization_gap": 0.0, "n_epochs": 1, "duration_seconds": 0.62}
+{"experiment_id": "87b6ab5abd3f", "agent": "marker_agent", "baseline_score": 0.4295714285714286, "best_score": 0.4295714285714286, "absolute_improvement": 0.0, "holdout_score": 0.42957142857142855, "generalization_gap": 5.551115123125783e-17, "n_epochs": 1, "duration_seconds": 0.72}
+{"experiment_id": "23c74d4ea257", "agent": "marker_agent", "baseline_score": 0.4295714285714286, "best_score": 0.5724285714285714, "absolute_improvement": 0.1428571428571428, "holdout_score": 0.5724285714285714, "generalization_gap": 0.0, "n_epochs": 1, "duration_seconds": 0.66}
+{"experiment_id": "970f28b959d2", "agent": "marker_agent", "baseline_score": 0.7152857142857143, "best_score": 0.7152857142857143, "absolute_improvement": 0.0, "holdout_score": 0.7152857142857143, "generalization_gap": 0.0, "n_epochs": 1, "duration_seconds": 0.67}
+{"experiment_id": "7d022a054e67", "agent": "marker_agent", "baseline_score": 0.8581428571428571, "best_score": 0.8581428571428571, "absolute_improvement": 0.0, "holdout_score": 0.8581428571428571, "generalization_gap": 0.0, "n_epochs": 1, "duration_seconds": 0.78}
+{"experiment_id": "16acabe00699", "agent": "marker_agent", "baseline_score": 1.0009999999999997, "best_score": 1.0009999999999997, "absolute_improvement": 0.0, "holdout_score": 1.001, "generalization_gap": -2.220446049250313e-16, "n_epochs": 1, "duration_seconds": 0.72}
+{"experiment_id": "142c287799f8", "agent": "marker_agent", "baseline_score": 1.0009999999999997, "best_score": 1.0009999999999997, "absolute_improvement": 0.0, "holdout_score": 1.001, "generalization_gap": -2.220446049250313e-16, "n_epochs": 1, "duration_seconds": 0.59}
+{"experiment_id": "a279cf052e84", "agent": "marker_agent", "baseline_score": 1.0009999999999997, "best_score": 1.0009999999999997, "absolute_improvement": 0.0, "holdout_score": 1.001, "generalization_gap": -2.220446049250313e-16, "n_epochs": 1, "duration_seconds": 1.08}
+{"experiment_id": "5e91ff17f6cc", "agent": "marker_agent", "baseline_score": 1.0009999999999997, "best_score": 1.0009999999999997, "absolute_improvement": 0.0, "holdout_score": 1.001, "generalization_gap": -2.220446049250313e-16, "n_epochs": 1, "duration_seconds": 0.76}
+{"experiment_id": "7af1cc2a1511", "agent": "marker_agent", "baseline_score": 0.001, "best_score": 0.14385714285714288, "absolute_improvement": 0.14285714285714288, "holdout_score": 0.14385714285714285, "generalization_gap": 2.7755575615628914e-17, "n_epochs": 1, "duration_seconds": 0.15}
+{"experiment_id": "736d05749772", "agent": "marker_agent", "baseline_score": 0.14385714285714288, "best_score": 0.14385714285714288, "absolute_improvement": 0.0, "holdout_score": 0.14385714285714285, "generalization_gap": 2.7755575615628914e-17, "n_epochs": 1, "duration_seconds": 0.19}
+{"experiment_id": "3d1f73e32b77", "agent": "marker_agent", "baseline_score": 0.14385714285714288, "best_score": 0.14385714285714288, "absolute_improvement": 0.0, "holdout_score": 0.14385714285714285, "generalization_gap": 2.7755575615628914e-17, "n_epochs": 1, "duration_seconds": 0.17}
+{"experiment_id": "46c20a9af339", "agent": "marker_agent", "baseline_score": 0.14385714285714288, "best_score": 0.14385714285714288, "absolute_improvement": 0.0, "holdout_score": 0.14385714285714285, "generalization_gap": 2.7755575615628914e-17, "n_epochs": 1, "duration_seconds": 0.12}
+{"experiment_id": "58f5a7c0ef7c", "agent": "marker_agent", "baseline_score": 0.14385714285714288, "best_score": 0.14385714285714288, "absolute_improvement": 0.0, "holdout_score": 0.14385714285714285, "generalization_gap": 2.7755575615628914e-17, "n_epochs": 1, "duration_seconds": 0.11}
+{"experiment_id": "af34d7408323", "agent": "marker_agent", "baseline_score": 0.001, "best_score": 0.14385714285714288, "absolute_improvement": 0.14285714285714288, "holdout_score": 0.14385714285714285, "generalization_gap": 2.7755575615628914e-17, "n_epochs": 1, "duration_seconds": 0.18}
+{"experiment_id": "8ef4613380b1", "agent": "marker_agent", "baseline_score": 0.14385714285714288, "best_score": 0.2867142857142857, "absolute_improvement": 0.14285714285714282, "holdout_score": 0.2867142857142857, "generalization_gap": 0.0, "n_epochs": 1, "duration_seconds": 0.15}
+{"experiment_id": "d01b1f182694", "agent": "marker_agent", "baseline_score": 0.4295714285714286, "best_score": 0.4295714285714286, "absolute_improvement": 0.0, "holdout_score": 0.42957142857142855, "generalization_gap": 5.551115123125783e-17, "n_epochs": 1, "duration_seconds": 0.13}
+{"experiment_id": "538677bc709e", "agent": "marker_agent", "baseline_score": 0.4295714285714286, "best_score": 0.4295714285714286, "absolute_improvement": 0.0, "holdout_score": 0.42957142857142855, "generalization_gap": 5.551115123125783e-17, "n_epochs": 1, "duration_seconds": 0.13}
+{"experiment_id": "fd94ff3bc2e5", "agent": "marker_agent", "baseline_score": 0.4295714285714286, "best_score": 0.4295714285714286, "absolute_improvement": 0.0, "holdout_score": 0.42957142857142855, "generalization_gap": 5.551115123125783e-17, "n_epochs": 1, "duration_seconds": 0.13}
+{"experiment_id": "b9d95379dd2f", "agent": "marker_agent", "baseline_score": 0.4295714285714286, "best_score": 0.4295714285714286, "absolute_improvement": 0.0, "holdout_score": 0.42957142857142855, "generalization_gap": 5.551115123125783e-17, "n_epochs": 1, "duration_seconds": 0.18}
+{"experiment_id": "d5d65ca1be6a", "agent": "marker_agent", "baseline_score": 0.4295714285714286, "best_score": 0.4295714285714286, "absolute_improvement": 0.0, "holdout_score": 0.42957142857142855, "generalization_gap": 5.551115123125783e-17, "n_epochs": 1, "duration_seconds": 0.12}
diff --git a/optimization_results/showcase/few_shot_bootstrap/config.json b/optimization_results/showcase/few_shot_bootstrap/config.json
new file mode 100644
index 0000000..a206226
--- /dev/null
+++ b/optimization_results/showcase/few_shot_bootstrap/config.json
@@ -0,0 +1,21 @@
+{
+ "system_prompt": "You are a helpful AI assistant.",
+ "few_shot_examples": [
+ {
+ "query": "What is the budget for Q3?",
+ "response": "BASE: answer to What is the budget for Q3?"
+ },
+ {
+ "query": "What did the customer ask for in the last meeting?",
+ "response": "BASE: answer to What did the customer ask for in the last meeting?"
+ },
+ {
+ "query": "Who approved the scope change?",
+ "response": "BASE: answer to Who approved the scope change?"
+ },
+ {
+ "query": "When is the next release candidate?",
+ "response": "BASE: answer to When is the next release candidate?"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/optimization_results/showcase/few_shot_bootstrap/evaluation_history.json b/optimization_results/showcase/few_shot_bootstrap/evaluation_history.json
new file mode 100644
index 0000000..137067a
--- /dev/null
+++ b/optimization_results/showcase/few_shot_bootstrap/evaluation_history.json
@@ -0,0 +1,147 @@
+[
+ {
+ "agent_name": "marker_agent",
+ "dataset_name": "inline",
+ "scores": {
+ "banana": 1.0,
+ "strawberry": 0.0,
+ "blueberry": 0.0,
+ "kiwi": 0.0,
+ "temp_ok": 0.0,
+ "r1": 0.0,
+ "r2": 0.0,
+ "r3": 0.0,
+ "quality": 0.14285714285714288,
+ "exact_key_match": 1.0
+ },
+ "num_examples": 4,
+ "pass_rate": 0.0,
+ "metadata": {},
+ "example_results": [
+ {
+ "example_id": "marker_016",
+ "prediction": {
+ "response": "OPT: answer to What did the customer ask for in the last meeting? banana",
+ "used_prompt": "You are a helpful AI assistant.\n\n## Few-shot examples\n\nExample 1\nQ: What is the budget for Q3?\nA: BASE: answer to What is the budget for Q3?\n\nExample 2\nQ: What did the customer ask for in the last meeting?\nA: BASE: answer to What did the customer ask for in the last meeting?\n\nExample 3\nQ: Who approved the scope change?\nA: BASE: answer to Who approved the scope change?\n\nExample 4\nQ: When is the next release candidate?\nA: BASE: answer to When is the next release candidate?",
+ "temperature": 0.7,
+ "temp_ok": false,
+ "temp_rungs": 0,
+ "banana_ok": true,
+ "strawberry_ok": false,
+ "blueberry_ok": false,
+ "kiwi_ok": false,
+ "n_satisfied": 1,
+ "difficulty": 1
+ },
+ "scores": {
+ "banana": 1.0,
+ "strawberry": 0.0,
+ "blueberry": 0.0,
+ "kiwi": 0.0,
+ "temp_ok": 0.0,
+ "r1": 0.0,
+ "r2": 0.0,
+ "r3": 0.0,
+ "quality": 0.14285714285714288,
+ "exact_key_match": 1.0
+ },
+ "duration_ms": 0.16948600000432634,
+ "error": null,
+ "metadata": {}
+ },
+ {
+ "example_id": "marker_017",
+ "prediction": {
+ "response": "OPT: answer to Which teams need to be unblocked this week? banana",
+ "used_prompt": "You are a helpful AI assistant.\n\n## Few-shot examples\n\nExample 1\nQ: What is the budget for Q3?\nA: BASE: answer to What is the budget for Q3?\n\nExample 2\nQ: What did the customer ask for in the last meeting?\nA: BASE: answer to What did the customer ask for in the last meeting?\n\nExample 3\nQ: Who approved the scope change?\nA: BASE: answer to Who approved the scope change?\n\nExample 4\nQ: When is the next release candidate?\nA: BASE: answer to When is the next release candidate?",
+ "temperature": 0.7,
+ "temp_ok": false,
+ "temp_rungs": 0,
+ "banana_ok": true,
+ "strawberry_ok": false,
+ "blueberry_ok": false,
+ "kiwi_ok": false,
+ "n_satisfied": 1,
+ "difficulty": 1
+ },
+ "scores": {
+ "banana": 1.0,
+ "strawberry": 0.0,
+ "blueberry": 0.0,
+ "kiwi": 0.0,
+ "temp_ok": 0.0,
+ "r1": 0.0,
+ "r2": 0.0,
+ "r3": 0.0,
+ "quality": 0.14285714285714288,
+ "exact_key_match": 1.0
+ },
+ "duration_ms": 0.10843399991244951,
+ "error": null,
+ "metadata": {}
+ },
+ {
+ "example_id": "marker_018",
+ "prediction": {
+ "response": "OPT: answer to What is the budget for Q3? banana",
+ "used_prompt": "You are a helpful AI assistant.\n\n## Few-shot examples\n\nExample 1\nQ: What is the budget for Q3?\nA: BASE: answer to What is the budget for Q3?\n\nExample 2\nQ: What did the customer ask for in the last meeting?\nA: BASE: answer to What did the customer ask for in the last meeting?\n\nExample 3\nQ: Who approved the scope change?\nA: BASE: answer to Who approved the scope change?\n\nExample 4\nQ: When is the next release candidate?\nA: BASE: answer to When is the next release candidate?",
+ "temperature": 0.7,
+ "temp_ok": false,
+ "temp_rungs": 0,
+ "banana_ok": true,
+ "strawberry_ok": false,
+ "blueberry_ok": false,
+ "kiwi_ok": false,
+ "n_satisfied": 1,
+ "difficulty": 1
+ },
+ "scores": {
+ "banana": 1.0,
+ "strawberry": 0.0,
+ "blueberry": 0.0,
+ "kiwi": 0.0,
+ "temp_ok": 0.0,
+ "r1": 0.0,
+ "r2": 0.0,
+ "r3": 0.0,
+ "quality": 0.14285714285714288,
+ "exact_key_match": 1.0
+ },
+ "duration_ms": 0.09667599988461006,
+ "error": null,
+ "metadata": {}
+ },
+ {
+ "example_id": "marker_019",
+ "prediction": {
+ "response": "OPT: answer to What is the current status of the project? banana",
+ "used_prompt": "You are a helpful AI assistant.\n\n## Few-shot examples\n\nExample 1\nQ: What is the budget for Q3?\nA: BASE: answer to What is the budget for Q3?\n\nExample 2\nQ: What did the customer ask for in the last meeting?\nA: BASE: answer to What did the customer ask for in the last meeting?\n\nExample 3\nQ: Who approved the scope change?\nA: BASE: answer to Who approved the scope change?\n\nExample 4\nQ: When is the next release candidate?\nA: BASE: answer to When is the next release candidate?",
+ "temperature": 0.7,
+ "temp_ok": false,
+ "temp_rungs": 0,
+ "banana_ok": true,
+ "strawberry_ok": false,
+ "blueberry_ok": false,
+ "kiwi_ok": false,
+ "n_satisfied": 1,
+ "difficulty": 1
+ },
+ "scores": {
+ "banana": 1.0,
+ "strawberry": 0.0,
+ "blueberry": 0.0,
+ "kiwi": 0.0,
+ "temp_ok": 0.0,
+ "r1": 0.0,
+ "r2": 0.0,
+ "r3": 0.0,
+ "quality": 0.14285714285714288,
+ "exact_key_match": 1.0
+ },
+ "duration_ms": 0.10697800007619662,
+ "error": null,
+ "metadata": {}
+ }
+ ]
+ }
+]
\ No newline at end of file
diff --git a/optimization_results/showcase/few_shot_bootstrap/fit_history.json b/optimization_results/showcase/few_shot_bootstrap/fit_history.json
new file mode 100644
index 0000000..c68b31f
--- /dev/null
+++ b/optimization_results/showcase/few_shot_bootstrap/fit_history.json
@@ -0,0 +1,249 @@
+{
+ "params": {
+ "epochs": 10,
+ "optimizer": "PromptFitterBridge",
+ "metrics": [
+ "banana",
+ "strawberry",
+ "blueberry",
+ "kiwi",
+ "temp_ok",
+ "r1",
+ "r2",
+ "r3",
+ "quality",
+ "exact_key_match"
+ ],
+ "loss": "quality_loss",
+ "train_size": 10,
+ "val_size": 6,
+ "optimize": {
+ "search_space": {
+ "optimize_system_prompt": false,
+ "optimize_user_template": false,
+ "optimize_few_shot": true,
+ "optimize_model_params": false,
+ "optimize_rag_params": false,
+ "optimize_tool_params": false,
+ "optimize_model_choice": false,
+ "model_param_space": {
+ "temperature": [
+ 0.0,
+ 0.1,
+ 0.2,
+ 0.4,
+ 0.7
+ ],
+ "top_p": [
+ 0.7,
+ 0.9,
+ 1.0
+ ],
+ "max_tokens": [
+ 800,
+ 1200,
+ 2000
+ ]
+ },
+ "rag_param_space": {},
+ "tool_param_space": {},
+ "model_choices": [],
+ "fallback_models": [],
+ "routing_weight_space": {},
+ "max_few_shot_examples": 5,
+ "few_shot_selection_strategy": "diversity_weighted",
+ "search_method": "random",
+ "optimize_nodes": [],
+ "node_match": null
+ },
+ "optimizer": "few_shot_bootstrap",
+ "max_trials": 6
+ }
+ },
+ "epoch": [
+ -1,
+ 0,
+ 1,
+ 2,
+ 3,
+ 4
+ ],
+ "history": {
+ "banana": [
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "strawberry": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "blueberry": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "kiwi": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "temp_ok": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "r1": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "r2": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "r3": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "quality": [
+ 0.0,
+ 0.14285714285714285,
+ 0.14285714285714285,
+ 0.14285714285714285,
+ 0.14285714285714285,
+ 0.14285714285714285
+ ],
+ "exact_key_match": [
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "loss": [
+ 1.0,
+ 0.857142857142857,
+ 0.857142857142857,
+ 0.857142857142857,
+ 0.857142857142857,
+ 0.857142857142857
+ ],
+ "val_banana": [
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_strawberry": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_blueberry": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_kiwi": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_temp_ok": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_r1": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_r2": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_r3": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_quality": [
+ 0.0,
+ 0.14285714285714288,
+ 0.14285714285714288,
+ 0.14285714285714288,
+ 0.14285714285714288,
+ 0.14285714285714288
+ ],
+ "val_exact_key_match": [
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_loss": [
+ 1.0,
+ 0.8571428571428571,
+ 0.8571428571428571,
+ 0.8571428571428571,
+ 0.8571428571428571,
+ 0.8571428571428571
+ ]
+ }
+}
\ No newline at end of file
diff --git a/optimization_results/showcase/few_shot_bootstrap/metadata.json b/optimization_results/showcase/few_shot_bootstrap/metadata.json
new file mode 100644
index 0000000..937638b
--- /dev/null
+++ b/optimization_results/showcase/few_shot_bootstrap/metadata.json
@@ -0,0 +1,21 @@
+{
+ "agent_class": "examples.keras_optimize_showcase.agent.MarkerAgent",
+ "agent_name": "marker_agent",
+ "agent_version": "1.0.0",
+ "dataset_name": "marker_fake",
+ "dataset_size": 20,
+ "metrics": [
+ "banana",
+ "strawberry",
+ "blueberry",
+ "kiwi",
+ "temp_ok",
+ "r1",
+ "r2",
+ "r3",
+ "quality",
+ "exact_key_match"
+ ],
+ "optimizer": "PromptFitterBridge",
+ "loss": "quality_loss"
+}
\ No newline at end of file
diff --git a/optimization_results/showcase/gepa_like/config.json b/optimization_results/showcase/gepa_like/config.json
new file mode 100644
index 0000000..328a48a
--- /dev/null
+++ b/optimization_results/showcase/gepa_like/config.json
@@ -0,0 +1,4 @@
+{
+ "system_prompt": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query.",
+ "few_shot_examples": []
+}
\ No newline at end of file
diff --git a/optimization_results/showcase/gepa_like/evaluation_history.json b/optimization_results/showcase/gepa_like/evaluation_history.json
new file mode 100644
index 0000000..86ea708
--- /dev/null
+++ b/optimization_results/showcase/gepa_like/evaluation_history.json
@@ -0,0 +1,147 @@
+[
+ {
+ "agent_name": "marker_agent",
+ "dataset_name": "inline",
+ "scores": {
+ "banana": 1.0,
+ "strawberry": 1.0,
+ "blueberry": 0.0,
+ "kiwi": 0.0,
+ "temp_ok": 0.0,
+ "r1": 0.0,
+ "r2": 0.0,
+ "r3": 0.0,
+ "quality": 0.28571428571428575,
+ "exact_key_match": 1.0
+ },
+ "num_examples": 4,
+ "pass_rate": 0.0,
+ "metadata": {},
+ "example_results": [
+ {
+ "example_id": "marker_016",
+ "prediction": {
+ "response": "PARTIAL: answer to What did the customer ask for in the last meeting? banana strawberry",
+ "used_prompt": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query.",
+ "temperature": 0.7,
+ "temp_ok": false,
+ "temp_rungs": 0,
+ "banana_ok": true,
+ "strawberry_ok": true,
+ "blueberry_ok": false,
+ "kiwi_ok": false,
+ "n_satisfied": 2,
+ "difficulty": 4
+ },
+ "scores": {
+ "banana": 1.0,
+ "strawberry": 1.0,
+ "blueberry": 0.0,
+ "kiwi": 0.0,
+ "temp_ok": 0.0,
+ "r1": 0.0,
+ "r2": 0.0,
+ "r3": 0.0,
+ "quality": 0.28571428571428575,
+ "exact_key_match": 1.0
+ },
+ "duration_ms": 0.20288800010348496,
+ "error": null,
+ "metadata": {}
+ },
+ {
+ "example_id": "marker_017",
+ "prediction": {
+ "response": "PARTIAL: answer to Which teams need to be unblocked this week? banana strawberry",
+ "used_prompt": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query.",
+ "temperature": 0.7,
+ "temp_ok": false,
+ "temp_rungs": 0,
+ "banana_ok": true,
+ "strawberry_ok": true,
+ "blueberry_ok": false,
+ "kiwi_ok": false,
+ "n_satisfied": 2,
+ "difficulty": 4
+ },
+ "scores": {
+ "banana": 1.0,
+ "strawberry": 1.0,
+ "blueberry": 0.0,
+ "kiwi": 0.0,
+ "temp_ok": 0.0,
+ "r1": 0.0,
+ "r2": 0.0,
+ "r3": 0.0,
+ "quality": 0.28571428571428575,
+ "exact_key_match": 1.0
+ },
+ "duration_ms": 0.1554619998387352,
+ "error": null,
+ "metadata": {}
+ },
+ {
+ "example_id": "marker_018",
+ "prediction": {
+ "response": "PARTIAL: answer to What is the budget for Q3? banana strawberry",
+ "used_prompt": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query.",
+ "temperature": 0.7,
+ "temp_ok": false,
+ "temp_rungs": 0,
+ "banana_ok": true,
+ "strawberry_ok": true,
+ "blueberry_ok": false,
+ "kiwi_ok": false,
+ "n_satisfied": 2,
+ "difficulty": 4
+ },
+ "scores": {
+ "banana": 1.0,
+ "strawberry": 1.0,
+ "blueberry": 0.0,
+ "kiwi": 0.0,
+ "temp_ok": 0.0,
+ "r1": 0.0,
+ "r2": 0.0,
+ "r3": 0.0,
+ "quality": 0.28571428571428575,
+ "exact_key_match": 1.0
+ },
+ "duration_ms": 0.14165600009619084,
+ "error": null,
+ "metadata": {}
+ },
+ {
+ "example_id": "marker_019",
+ "prediction": {
+ "response": "PARTIAL: answer to What is the current status of the project? banana strawberry",
+ "used_prompt": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query.",
+ "temperature": 0.7,
+ "temp_ok": false,
+ "temp_rungs": 0,
+ "banana_ok": true,
+ "strawberry_ok": true,
+ "blueberry_ok": false,
+ "kiwi_ok": false,
+ "n_satisfied": 2,
+ "difficulty": 4
+ },
+ "scores": {
+ "banana": 1.0,
+ "strawberry": 1.0,
+ "blueberry": 0.0,
+ "kiwi": 0.0,
+ "temp_ok": 0.0,
+ "r1": 0.0,
+ "r2": 0.0,
+ "r3": 0.0,
+ "quality": 0.28571428571428575,
+ "exact_key_match": 1.0
+ },
+ "duration_ms": 0.13400100010585447,
+ "error": null,
+ "metadata": {}
+ }
+ ]
+ }
+]
\ No newline at end of file
diff --git a/optimization_results/showcase/gepa_like/fit_history.json b/optimization_results/showcase/gepa_like/fit_history.json
new file mode 100644
index 0000000..1b3f96a
--- /dev/null
+++ b/optimization_results/showcase/gepa_like/fit_history.json
@@ -0,0 +1,272 @@
+{
+ "params": {
+ "epochs": 10,
+ "optimizer": "PromptFitterBridge",
+ "metrics": [
+ "banana",
+ "strawberry",
+ "blueberry",
+ "kiwi",
+ "temp_ok",
+ "r1",
+ "r2",
+ "r3",
+ "quality",
+ "exact_key_match"
+ ],
+ "loss": "quality_loss",
+ "train_size": 10,
+ "val_size": 6,
+ "optimize": {
+ "search_space": {
+ "optimize_system_prompt": true,
+ "optimize_user_template": false,
+ "optimize_few_shot": false,
+ "optimize_model_params": false,
+ "optimize_rag_params": false,
+ "optimize_tool_params": false,
+ "optimize_model_choice": false,
+ "model_param_space": {
+ "temperature": [
+ 0.0,
+ 0.1,
+ 0.2,
+ 0.4,
+ 0.7
+ ],
+ "top_p": [
+ 0.7,
+ 0.9,
+ 1.0
+ ],
+ "max_tokens": [
+ 800,
+ 1200,
+ 2000
+ ]
+ },
+ "rag_param_space": {},
+ "tool_param_space": {},
+ "model_choices": [],
+ "fallback_models": [],
+ "routing_weight_space": {},
+ "max_few_shot_examples": 5,
+ "few_shot_selection_strategy": "diversity_weighted",
+ "search_method": "random",
+ "optimize_nodes": [],
+ "node_match": null
+ },
+ "optimizer": "gepa_like",
+ "max_trials": 6
+ }
+ },
+ "epoch": [
+ -1,
+ 0,
+ 1,
+ 2,
+ 3,
+ 4,
+ 5
+ ],
+ "history": {
+ "banana": [
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "strawberry": [
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "blueberry": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "kiwi": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "temp_ok": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "r1": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "r2": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "r3": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "quality": [
+ 0.0,
+ 0.14285714285714285,
+ 0.2857142857142857,
+ 0.2857142857142857,
+ 0.2857142857142857,
+ 0.2857142857142857,
+ 0.2857142857142857
+ ],
+ "exact_key_match": [
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "loss": [
+ 1.0,
+ 0.857142857142857,
+ 0.7142857142857143,
+ 0.7142857142857143,
+ 0.7142857142857143,
+ 0.7142857142857143,
+ 0.7142857142857143
+ ],
+ "val_banana": [
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_strawberry": [
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_blueberry": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_kiwi": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_temp_ok": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_r1": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_r2": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_r3": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_quality": [
+ 0.0,
+ 0.14285714285714288,
+ 0.28571428571428575,
+ 0.28571428571428575,
+ 0.28571428571428575,
+ 0.28571428571428575,
+ 0.28571428571428575
+ ],
+ "val_exact_key_match": [
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_loss": [
+ 1.0,
+ 0.8571428571428571,
+ 0.7142857142857143,
+ 0.7142857142857143,
+ 0.7142857142857143,
+ 0.7142857142857143,
+ 0.7142857142857143
+ ]
+ }
+}
\ No newline at end of file
diff --git a/optimization_results/showcase/gepa_like/metadata.json b/optimization_results/showcase/gepa_like/metadata.json
new file mode 100644
index 0000000..937638b
--- /dev/null
+++ b/optimization_results/showcase/gepa_like/metadata.json
@@ -0,0 +1,21 @@
+{
+ "agent_class": "examples.keras_optimize_showcase.agent.MarkerAgent",
+ "agent_name": "marker_agent",
+ "agent_version": "1.0.0",
+ "dataset_name": "marker_fake",
+ "dataset_size": 20,
+ "metrics": [
+ "banana",
+ "strawberry",
+ "blueberry",
+ "kiwi",
+ "temp_ok",
+ "r1",
+ "r2",
+ "r3",
+ "quality",
+ "exact_key_match"
+ ],
+ "optimizer": "PromptFitterBridge",
+ "loss": "quality_loss"
+}
\ No newline at end of file
diff --git a/optimization_results/showcase/mipro_like/config.json b/optimization_results/showcase/mipro_like/config.json
new file mode 100644
index 0000000..e737c6a
--- /dev/null
+++ b/optimization_results/showcase/mipro_like/config.json
@@ -0,0 +1,22 @@
+{
+ "system_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query.",
+ "few_shot_examples": [
+ {
+ "query": "What happened at the steering committee?",
+ "response": "PARTIAL: answer to What happened at the steering committee? banana strawberry blueberry"
+ },
+ {
+ "query": "What follow-up is needed after the demo?",
+ "response": "PARTIAL: answer to What follow-up is needed after the demo? banana strawberry blueberry"
+ },
+ {
+ "query": "What is the budget for Q3?",
+ "response": "PARTIAL: answer to What is the budget for Q3? banana strawberry blueberry"
+ },
+ {
+ "query": "When is the next release candidate?",
+ "response": "PARTIAL: answer to When is the next release candidate? banana strawberry blueberry"
+ }
+ ],
+ "temperature": 0.15
+}
\ No newline at end of file
diff --git a/optimization_results/showcase/mipro_like/evaluation_history.json b/optimization_results/showcase/mipro_like/evaluation_history.json
new file mode 100644
index 0000000..f0bc8a3
--- /dev/null
+++ b/optimization_results/showcase/mipro_like/evaluation_history.json
@@ -0,0 +1,147 @@
+[
+ {
+ "agent_name": "marker_agent",
+ "dataset_name": "inline",
+ "scores": {
+ "banana": 1.0,
+ "strawberry": 1.0,
+ "blueberry": 1.0,
+ "kiwi": 1.0,
+ "temp_ok": 1.0,
+ "r1": 1.0,
+ "r2": 1.0,
+ "r3": 1.0,
+ "quality": 1.0,
+ "exact_key_match": 1.0
+ },
+ "num_examples": 4,
+ "pass_rate": 1.0,
+ "metadata": {},
+ "example_results": [
+ {
+ "example_id": "marker_016",
+ "prediction": {
+ "response": "OPT: answer to What did the customer ask for in the last meeting? banana strawberry blueberry kiwi r1 r2 r3",
+ "used_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query.\n\n## Few-shot examples\n\nExample 1\nQ: What happened at the steering committee?\nA: PARTIAL: answer to What happened at the steering committee? banana strawberry blueberry\n\nExample 2\nQ: What follow-up is needed after the demo?\nA: PARTIAL: answer to What follow-up is needed after the demo? banana strawberry blueberry\n\nExample 3\nQ: What is the budget for Q3?\nA: PARTIAL: answer to What is the budget for Q3? banana strawberry blueberry\n\nExample 4\nQ: When is the next release candidate?\nA: PARTIAL: answer to When is the next release candidate? banana strawberry blueberry",
+ "temperature": 0.15,
+ "temp_ok": true,
+ "temp_rungs": 3,
+ "banana_ok": true,
+ "strawberry_ok": true,
+ "blueberry_ok": true,
+ "kiwi_ok": true,
+ "n_satisfied": 7,
+ "difficulty": 7
+ },
+ "scores": {
+ "banana": 1.0,
+ "strawberry": 1.0,
+ "blueberry": 1.0,
+ "kiwi": 1.0,
+ "temp_ok": 1.0,
+ "r1": 1.0,
+ "r2": 1.0,
+ "r3": 1.0,
+ "quality": 1.0,
+ "exact_key_match": 1.0
+ },
+ "duration_ms": 0.2729759999056114,
+ "error": null,
+ "metadata": {}
+ },
+ {
+ "example_id": "marker_017",
+ "prediction": {
+ "response": "OPT: answer to Which teams need to be unblocked this week? banana strawberry blueberry kiwi r1 r2 r3",
+ "used_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query.\n\n## Few-shot examples\n\nExample 1\nQ: What happened at the steering committee?\nA: PARTIAL: answer to What happened at the steering committee? banana strawberry blueberry\n\nExample 2\nQ: What follow-up is needed after the demo?\nA: PARTIAL: answer to What follow-up is needed after the demo? banana strawberry blueberry\n\nExample 3\nQ: What is the budget for Q3?\nA: PARTIAL: answer to What is the budget for Q3? banana strawberry blueberry\n\nExample 4\nQ: When is the next release candidate?\nA: PARTIAL: answer to When is the next release candidate? banana strawberry blueberry",
+ "temperature": 0.15,
+ "temp_ok": true,
+ "temp_rungs": 3,
+ "banana_ok": true,
+ "strawberry_ok": true,
+ "blueberry_ok": true,
+ "kiwi_ok": true,
+ "n_satisfied": 7,
+ "difficulty": 7
+ },
+ "scores": {
+ "banana": 1.0,
+ "strawberry": 1.0,
+ "blueberry": 1.0,
+ "kiwi": 1.0,
+ "temp_ok": 1.0,
+ "r1": 1.0,
+ "r2": 1.0,
+ "r3": 1.0,
+ "quality": 1.0,
+ "exact_key_match": 1.0
+ },
+ "duration_ms": 0.16014500010896882,
+ "error": null,
+ "metadata": {}
+ },
+ {
+ "example_id": "marker_018",
+ "prediction": {
+ "response": "OPT: answer to What is the budget for Q3? banana strawberry blueberry kiwi r1 r2 r3",
+ "used_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query.\n\n## Few-shot examples\n\nExample 1\nQ: What happened at the steering committee?\nA: PARTIAL: answer to What happened at the steering committee? banana strawberry blueberry\n\nExample 2\nQ: What follow-up is needed after the demo?\nA: PARTIAL: answer to What follow-up is needed after the demo? banana strawberry blueberry\n\nExample 3\nQ: What is the budget for Q3?\nA: PARTIAL: answer to What is the budget for Q3? banana strawberry blueberry\n\nExample 4\nQ: When is the next release candidate?\nA: PARTIAL: answer to When is the next release candidate? banana strawberry blueberry",
+ "temperature": 0.15,
+ "temp_ok": true,
+ "temp_rungs": 3,
+ "banana_ok": true,
+ "strawberry_ok": true,
+ "blueberry_ok": true,
+ "kiwi_ok": true,
+ "n_satisfied": 7,
+ "difficulty": 7
+ },
+ "scores": {
+ "banana": 1.0,
+ "strawberry": 1.0,
+ "blueberry": 1.0,
+ "kiwi": 1.0,
+ "temp_ok": 1.0,
+ "r1": 1.0,
+ "r2": 1.0,
+ "r3": 1.0,
+ "quality": 1.0,
+ "exact_key_match": 1.0
+ },
+ "duration_ms": 0.16184500009330804,
+ "error": null,
+ "metadata": {}
+ },
+ {
+ "example_id": "marker_019",
+ "prediction": {
+ "response": "OPT: answer to What is the current status of the project? banana strawberry blueberry kiwi r1 r2 r3",
+ "used_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query.\n\n## Few-shot examples\n\nExample 1\nQ: What happened at the steering committee?\nA: PARTIAL: answer to What happened at the steering committee? banana strawberry blueberry\n\nExample 2\nQ: What follow-up is needed after the demo?\nA: PARTIAL: answer to What follow-up is needed after the demo? banana strawberry blueberry\n\nExample 3\nQ: What is the budget for Q3?\nA: PARTIAL: answer to What is the budget for Q3? banana strawberry blueberry\n\nExample 4\nQ: When is the next release candidate?\nA: PARTIAL: answer to When is the next release candidate? banana strawberry blueberry",
+ "temperature": 0.15,
+ "temp_ok": true,
+ "temp_rungs": 3,
+ "banana_ok": true,
+ "strawberry_ok": true,
+ "blueberry_ok": true,
+ "kiwi_ok": true,
+ "n_satisfied": 7,
+ "difficulty": 7
+ },
+ "scores": {
+ "banana": 1.0,
+ "strawberry": 1.0,
+ "blueberry": 1.0,
+ "kiwi": 1.0,
+ "temp_ok": 1.0,
+ "r1": 1.0,
+ "r2": 1.0,
+ "r3": 1.0,
+ "quality": 1.0,
+ "exact_key_match": 1.0
+ },
+ "duration_ms": 0.1364820000162581,
+ "error": null,
+ "metadata": {}
+ }
+ ]
+ }
+]
\ No newline at end of file
diff --git a/optimization_results/showcase/mipro_like/fit_history.json b/optimization_results/showcase/mipro_like/fit_history.json
new file mode 100644
index 0000000..85d7ac9
--- /dev/null
+++ b/optimization_results/showcase/mipro_like/fit_history.json
@@ -0,0 +1,365 @@
+{
+ "params": {
+ "epochs": 10,
+ "optimizer": "PromptFitterBridge",
+ "metrics": [
+ "banana",
+ "strawberry",
+ "blueberry",
+ "kiwi",
+ "temp_ok",
+ "r1",
+ "r2",
+ "r3",
+ "quality",
+ "exact_key_match"
+ ],
+ "loss": "quality_loss",
+ "train_size": 10,
+ "val_size": 6,
+ "optimize": {
+ "search_space": {
+ "optimize_system_prompt": true,
+ "optimize_user_template": false,
+ "optimize_few_shot": true,
+ "optimize_model_params": true,
+ "optimize_rag_params": false,
+ "optimize_tool_params": false,
+ "optimize_model_choice": false,
+ "model_param_space": {
+ "temperature": [
+ 0.7,
+ 0.65,
+ 0.6,
+ 0.55,
+ 0.5,
+ 0.45,
+ 0.4,
+ 0.35,
+ 0.3,
+ 0.25,
+ 0.2,
+ 0.15,
+ 0.1,
+ 0.05,
+ 0.02,
+ 0.0
+ ]
+ },
+ "rag_param_space": {},
+ "tool_param_space": {},
+ "model_choices": [],
+ "fallback_models": [],
+ "routing_weight_space": {},
+ "max_few_shot_examples": 5,
+ "few_shot_selection_strategy": "diversity_weighted",
+ "search_method": "random",
+ "optimize_nodes": [],
+ "node_match": null
+ },
+ "optimizer": "mipro_like",
+ "max_trials": 6
+ }
+ },
+ "epoch": [
+ -1,
+ 0,
+ 1,
+ 2,
+ 3,
+ 4,
+ 5,
+ 6,
+ 7,
+ 8,
+ 9
+ ],
+ "history": {
+ "banana": [
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "strawberry": [
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "blueberry": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "kiwi": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "temp_ok": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "r1": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "r2": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "r3": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "quality": [
+ 0.0,
+ 0.14285714285714285,
+ 0.2857142857142857,
+ 0.4285714285714287,
+ 0.5714285714285714,
+ 0.7142857142857143,
+ 0.8571428571428574,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "exact_key_match": [
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "loss": [
+ 1.0,
+ 0.857142857142857,
+ 0.7142857142857143,
+ 0.5714285714285713,
+ 0.4285714285714285,
+ 0.28571428571428564,
+ 0.1428571428571428,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_banana": [
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_strawberry": [
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_blueberry": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_kiwi": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_temp_ok": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_r1": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_r2": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_r3": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_quality": [
+ 0.0,
+ 0.14285714285714288,
+ 0.28571428571428575,
+ 0.42857142857142866,
+ 0.5714285714285715,
+ 0.7142857142857143,
+ 0.8571428571428573,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_exact_key_match": [
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_loss": [
+ 1.0,
+ 0.8571428571428571,
+ 0.7142857142857143,
+ 0.5714285714285713,
+ 0.42857142857142844,
+ 0.28571428571428564,
+ 0.1428571428571428,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ]
+ }
+}
\ No newline at end of file
diff --git a/optimization_results/showcase/mipro_like/metadata.json b/optimization_results/showcase/mipro_like/metadata.json
new file mode 100644
index 0000000..937638b
--- /dev/null
+++ b/optimization_results/showcase/mipro_like/metadata.json
@@ -0,0 +1,21 @@
+{
+ "agent_class": "examples.keras_optimize_showcase.agent.MarkerAgent",
+ "agent_name": "marker_agent",
+ "agent_version": "1.0.0",
+ "dataset_name": "marker_fake",
+ "dataset_size": 20,
+ "metrics": [
+ "banana",
+ "strawberry",
+ "blueberry",
+ "kiwi",
+ "temp_ok",
+ "r1",
+ "r2",
+ "r3",
+ "quality",
+ "exact_key_match"
+ ],
+ "optimizer": "PromptFitterBridge",
+ "loss": "quality_loss"
+}
\ No newline at end of file
diff --git a/optimization_results/showcase/param_search/config.json b/optimization_results/showcase/param_search/config.json
new file mode 100644
index 0000000..ec743a0
--- /dev/null
+++ b/optimization_results/showcase/param_search/config.json
@@ -0,0 +1,5 @@
+{
+ "system_prompt": "You are a helpful AI assistant.",
+ "few_shot_examples": [],
+ "temperature": 0.05
+}
\ No newline at end of file
diff --git a/optimization_results/showcase/param_search/evaluation_history.json b/optimization_results/showcase/param_search/evaluation_history.json
new file mode 100644
index 0000000..4ba0800
--- /dev/null
+++ b/optimization_results/showcase/param_search/evaluation_history.json
@@ -0,0 +1,147 @@
+[
+ {
+ "agent_name": "marker_agent",
+ "dataset_name": "inline",
+ "scores": {
+ "banana": 0.0,
+ "strawberry": 0.0,
+ "blueberry": 0.0,
+ "kiwi": 0.0,
+ "temp_ok": 1.0,
+ "r1": 1.0,
+ "r2": 1.0,
+ "r3": 1.0,
+ "quality": 0.42857142857142866,
+ "exact_key_match": 1.0
+ },
+ "num_examples": 4,
+ "pass_rate": 0.0,
+ "metadata": {},
+ "example_results": [
+ {
+ "example_id": "marker_016",
+ "prediction": {
+ "response": "PARTIAL: answer to What did the customer ask for in the last meeting? r1 r2 r3",
+ "used_prompt": "You are a helpful AI assistant.",
+ "temperature": 0.05,
+ "temp_ok": true,
+ "temp_rungs": 3,
+ "banana_ok": false,
+ "strawberry_ok": false,
+ "blueberry_ok": false,
+ "kiwi_ok": false,
+ "n_satisfied": 3,
+ "difficulty": 7
+ },
+ "scores": {
+ "banana": 0.0,
+ "strawberry": 0.0,
+ "blueberry": 0.0,
+ "kiwi": 0.0,
+ "temp_ok": 1.0,
+ "r1": 1.0,
+ "r2": 1.0,
+ "r3": 1.0,
+ "quality": 0.42857142857142866,
+ "exact_key_match": 1.0
+ },
+ "duration_ms": 0.20074999997632403,
+ "error": null,
+ "metadata": {}
+ },
+ {
+ "example_id": "marker_017",
+ "prediction": {
+ "response": "PARTIAL: answer to Which teams need to be unblocked this week? r1 r2 r3",
+ "used_prompt": "You are a helpful AI assistant.",
+ "temperature": 0.05,
+ "temp_ok": true,
+ "temp_rungs": 3,
+ "banana_ok": false,
+ "strawberry_ok": false,
+ "blueberry_ok": false,
+ "kiwi_ok": false,
+ "n_satisfied": 3,
+ "difficulty": 7
+ },
+ "scores": {
+ "banana": 0.0,
+ "strawberry": 0.0,
+ "blueberry": 0.0,
+ "kiwi": 0.0,
+ "temp_ok": 1.0,
+ "r1": 1.0,
+ "r2": 1.0,
+ "r3": 1.0,
+ "quality": 0.42857142857142866,
+ "exact_key_match": 1.0
+ },
+ "duration_ms": 0.21136499981366796,
+ "error": null,
+ "metadata": {}
+ },
+ {
+ "example_id": "marker_018",
+ "prediction": {
+ "response": "PARTIAL: answer to What is the budget for Q3? r1 r2 r3",
+ "used_prompt": "You are a helpful AI assistant.",
+ "temperature": 0.05,
+ "temp_ok": true,
+ "temp_rungs": 3,
+ "banana_ok": false,
+ "strawberry_ok": false,
+ "blueberry_ok": false,
+ "kiwi_ok": false,
+ "n_satisfied": 3,
+ "difficulty": 7
+ },
+ "scores": {
+ "banana": 0.0,
+ "strawberry": 0.0,
+ "blueberry": 0.0,
+ "kiwi": 0.0,
+ "temp_ok": 1.0,
+ "r1": 1.0,
+ "r2": 1.0,
+ "r3": 1.0,
+ "quality": 0.42857142857142866,
+ "exact_key_match": 1.0
+ },
+ "duration_ms": 0.1616590000139695,
+ "error": null,
+ "metadata": {}
+ },
+ {
+ "example_id": "marker_019",
+ "prediction": {
+ "response": "PARTIAL: answer to What is the current status of the project? r1 r2 r3",
+ "used_prompt": "You are a helpful AI assistant.",
+ "temperature": 0.05,
+ "temp_ok": true,
+ "temp_rungs": 3,
+ "banana_ok": false,
+ "strawberry_ok": false,
+ "blueberry_ok": false,
+ "kiwi_ok": false,
+ "n_satisfied": 3,
+ "difficulty": 7
+ },
+ "scores": {
+ "banana": 0.0,
+ "strawberry": 0.0,
+ "blueberry": 0.0,
+ "kiwi": 0.0,
+ "temp_ok": 1.0,
+ "r1": 1.0,
+ "r2": 1.0,
+ "r3": 1.0,
+ "quality": 0.42857142857142866,
+ "exact_key_match": 1.0
+ },
+ "duration_ms": 0.19959599990215793,
+ "error": null,
+ "metadata": {}
+ }
+ ]
+ }
+]
\ No newline at end of file
diff --git a/optimization_results/showcase/param_search/fit_history.json b/optimization_results/showcase/param_search/fit_history.json
new file mode 100644
index 0000000..330833d
--- /dev/null
+++ b/optimization_results/showcase/param_search/fit_history.json
@@ -0,0 +1,296 @@
+{
+ "params": {
+ "epochs": 10,
+ "optimizer": "PromptFitterBridge",
+ "metrics": [
+ "banana",
+ "strawberry",
+ "blueberry",
+ "kiwi",
+ "temp_ok",
+ "r1",
+ "r2",
+ "r3",
+ "quality",
+ "exact_key_match"
+ ],
+ "loss": "quality_loss",
+ "train_size": 10,
+ "val_size": 6,
+ "optimize": {
+ "search_space": {
+ "optimize_system_prompt": false,
+ "optimize_user_template": false,
+ "optimize_few_shot": false,
+ "optimize_model_params": true,
+ "optimize_rag_params": false,
+ "optimize_tool_params": false,
+ "optimize_model_choice": false,
+ "model_param_space": {
+ "temperature": [
+ 0.7,
+ 0.65,
+ 0.6,
+ 0.55,
+ 0.5,
+ 0.45,
+ 0.4,
+ 0.35,
+ 0.3,
+ 0.25,
+ 0.2,
+ 0.15,
+ 0.1,
+ 0.05,
+ 0.02,
+ 0.0
+ ]
+ },
+ "rag_param_space": {},
+ "tool_param_space": {},
+ "model_choices": [],
+ "fallback_models": [],
+ "routing_weight_space": {},
+ "max_few_shot_examples": 5,
+ "few_shot_selection_strategy": "diversity_weighted",
+ "search_method": "random",
+ "optimize_nodes": [],
+ "node_match": null
+ },
+ "optimizer": "param_search",
+ "max_trials": 6
+ }
+ },
+ "epoch": [
+ -1,
+ 0,
+ 1,
+ 2,
+ 3,
+ 4,
+ 5,
+ 6
+ ],
+ "history": {
+ "banana": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "strawberry": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "blueberry": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "kiwi": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "temp_ok": [
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "r1": [
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "r2": [
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "r3": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "quality": [
+ 0.0,
+ 0.14285714285714285,
+ 0.2857142857142857,
+ 0.4285714285714287,
+ 0.4285714285714287,
+ 0.4285714285714287,
+ 0.4285714285714287,
+ 0.4285714285714287
+ ],
+ "exact_key_match": [
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "loss": [
+ 1.0,
+ 0.857142857142857,
+ 0.7142857142857143,
+ 0.5714285714285713,
+ 0.5714285714285713,
+ 0.5714285714285713,
+ 0.5714285714285713,
+ 0.5714285714285713
+ ],
+ "val_banana": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_strawberry": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_blueberry": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_kiwi": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_temp_ok": [
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_r1": [
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_r2": [
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_r3": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_quality": [
+ 0.0,
+ 0.14285714285714288,
+ 0.28571428571428575,
+ 0.42857142857142866,
+ 0.42857142857142866,
+ 0.42857142857142866,
+ 0.42857142857142866,
+ 0.42857142857142866
+ ],
+ "val_exact_key_match": [
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_loss": [
+ 1.0,
+ 0.8571428571428571,
+ 0.7142857142857143,
+ 0.5714285714285713,
+ 0.5714285714285713,
+ 0.5714285714285713,
+ 0.5714285714285713,
+ 0.5714285714285713
+ ]
+ }
+}
\ No newline at end of file
diff --git a/optimization_results/showcase/param_search/metadata.json b/optimization_results/showcase/param_search/metadata.json
new file mode 100644
index 0000000..937638b
--- /dev/null
+++ b/optimization_results/showcase/param_search/metadata.json
@@ -0,0 +1,21 @@
+{
+ "agent_class": "examples.keras_optimize_showcase.agent.MarkerAgent",
+ "agent_name": "marker_agent",
+ "agent_version": "1.0.0",
+ "dataset_name": "marker_fake",
+ "dataset_size": 20,
+ "metrics": [
+ "banana",
+ "strawberry",
+ "blueberry",
+ "kiwi",
+ "temp_ok",
+ "r1",
+ "r2",
+ "r3",
+ "quality",
+ "exact_key_match"
+ ],
+ "optimizer": "PromptFitterBridge",
+ "loss": "quality_loss"
+}
\ No newline at end of file
diff --git a/optimization_results/showcase/reports/fit_few_shot_bootstrap.html b/optimization_results/showcase/reports/fit_few_shot_bootstrap.html
new file mode 100644
index 0000000..4850d90
--- /dev/null
+++ b/optimization_results/showcase/reports/fit_few_shot_bootstrap.html
@@ -0,0 +1,332 @@
+
+
+
+
+
+ PromptFitter Report
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/optimization_results/showcase/reports/fit_gepa_like.html b/optimization_results/showcase/reports/fit_gepa_like.html
new file mode 100644
index 0000000..7638ffb
--- /dev/null
+++ b/optimization_results/showcase/reports/fit_gepa_like.html
@@ -0,0 +1,332 @@
+
+
+
+
+
+ PromptFitter Report
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/optimization_results/showcase/reports/fit_mipro_like.html b/optimization_results/showcase/reports/fit_mipro_like.html
new file mode 100644
index 0000000..97a9062
--- /dev/null
+++ b/optimization_results/showcase/reports/fit_mipro_like.html
@@ -0,0 +1,332 @@
+
+
+
+
+
+ PromptFitter Report
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/optimization_results/showcase/reports/fit_param_search.html b/optimization_results/showcase/reports/fit_param_search.html
new file mode 100644
index 0000000..9a87a76
--- /dev/null
+++ b/optimization_results/showcase/reports/fit_param_search.html
@@ -0,0 +1,332 @@
+
+
+
+
+
+ PromptFitter Report
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/optimization_results/showcase/reports/fit_rewrite.html b/optimization_results/showcase/reports/fit_rewrite.html
new file mode 100644
index 0000000..197f2e4
--- /dev/null
+++ b/optimization_results/showcase/reports/fit_rewrite.html
@@ -0,0 +1,332 @@
+
+
+
+
+
+ PromptFitter Report
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/optimization_results/showcase/reports/summary_few_shot_bootstrap.json b/optimization_results/showcase/reports/summary_few_shot_bootstrap.json
new file mode 100644
index 0000000..d4139dd
--- /dev/null
+++ b/optimization_results/showcase/reports/summary_few_shot_bootstrap.json
@@ -0,0 +1,373 @@
+{
+ "mode": "few_shot_bootstrap",
+ "model": "omlx/Qwen3.5-9B-MLX-4bit",
+ "history": {
+ "params": {
+ "epochs": 10,
+ "optimizer": "PromptFitterBridge",
+ "metrics": [
+ "banana",
+ "strawberry",
+ "blueberry",
+ "kiwi",
+ "temp_ok",
+ "r1",
+ "r2",
+ "r3",
+ "quality",
+ "exact_key_match"
+ ],
+ "loss": "quality_loss",
+ "train_size": 10,
+ "val_size": 6,
+ "optimize": {
+ "search_space": {
+ "optimize_system_prompt": false,
+ "optimize_user_template": false,
+ "optimize_few_shot": true,
+ "optimize_model_params": false,
+ "optimize_rag_params": false,
+ "optimize_tool_params": false,
+ "optimize_model_choice": false,
+ "model_param_space": {
+ "temperature": [
+ 0.0,
+ 0.1,
+ 0.2,
+ 0.4,
+ 0.7
+ ],
+ "top_p": [
+ 0.7,
+ 0.9,
+ 1.0
+ ],
+ "max_tokens": [
+ 800,
+ 1200,
+ 2000
+ ]
+ },
+ "rag_param_space": {},
+ "tool_param_space": {},
+ "model_choices": [],
+ "fallback_models": [],
+ "routing_weight_space": {},
+ "max_few_shot_examples": 5,
+ "few_shot_selection_strategy": "diversity_weighted",
+ "search_method": "random",
+ "optimize_nodes": [],
+ "node_match": null
+ },
+ "optimizer": "few_shot_bootstrap",
+ "max_trials": 6
+ }
+ },
+ "epoch": [
+ -1,
+ 0,
+ 1,
+ 2,
+ 3,
+ 4
+ ],
+ "history": {
+ "banana": [
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "strawberry": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "blueberry": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "kiwi": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "temp_ok": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "r1": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "r2": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "r3": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "quality": [
+ 0.0,
+ 0.14285714285714285,
+ 0.14285714285714285,
+ 0.14285714285714285,
+ 0.14285714285714285,
+ 0.14285714285714285
+ ],
+ "exact_key_match": [
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "loss": [
+ 1.0,
+ 0.857142857142857,
+ 0.857142857142857,
+ 0.857142857142857,
+ 0.857142857142857,
+ 0.857142857142857
+ ],
+ "val_banana": [
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_strawberry": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_blueberry": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_kiwi": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_temp_ok": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_r1": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_r2": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_r3": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_quality": [
+ 0.0,
+ 0.14285714285714288,
+ 0.14285714285714288,
+ 0.14285714285714288,
+ 0.14285714285714288,
+ 0.14285714285714288
+ ],
+ "val_exact_key_match": [
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_loss": [
+ 1.0,
+ 0.8571428571428571,
+ 0.8571428571428571,
+ 0.8571428571428571,
+ 0.8571428571428571,
+ 0.8571428571428571
+ ]
+ }
+ },
+ "compiled_config": {
+ "system_prompt": "You are a helpful AI assistant.",
+ "few_shot_examples": "[{'query': 'What is the budget for Q3?', 'response': 'BASE: answer to What is the budget for Q3?'}, {'query': 'What did the customer ask for in the last meeting?', 'response': 'BASE: answer to What did the customer ask for in the last meeting?'}, {'query': 'Who approved the scope change?', 'response': 'BASE: answer to Who approved the scope change?'}, {'query': 'When is the next release candidate?', 'response': 'BASE: answer to When is the next release candidate?'}]"
+ },
+ "per_epoch_changes": [
+ {
+ "epoch": 0,
+ "loss": 0.857143,
+ "val_loss": 0.857143,
+ "improvement": null,
+ "changes": {
+ "prompt_diff": [
+ "--- before",
+ "+++ after",
+ "@@ -1 +1 @@",
+ "-You are a vague assistant.",
+ "+You are a helpful AI assistant."
+ ],
+ "params": {
+ "few_shot_examples": {
+ "old": null,
+ "new": [
+ {
+ "query": "What is the budget for Q3?",
+ "response": "BASE: answer to What is the budget for Q3?"
+ },
+ {
+ "query": "What did the customer ask for in the last meeting?",
+ "response": "BASE: answer to What did the customer ask for in the last meeting?"
+ },
+ {
+ "query": "Who approved the scope change?",
+ "response": "BASE: answer to Who approved the scope change?"
+ },
+ {
+ "query": "When is the next release candidate?",
+ "response": "BASE: answer to When is the next release candidate?"
+ }
+ ]
+ }
+ }
+ },
+ "prompt": "You are a helpful AI assistant."
+ },
+ {
+ "epoch": 1,
+ "loss": 0.857143,
+ "val_loss": 0.857143,
+ "improvement": 0.0,
+ "changes": {
+ "prompt_diff": [],
+ "params": {}
+ },
+ "prompt": "You are a helpful AI assistant."
+ },
+ {
+ "epoch": 2,
+ "loss": 0.857143,
+ "val_loss": 0.857143,
+ "improvement": 0.0,
+ "changes": {
+ "prompt_diff": [],
+ "params": {}
+ },
+ "prompt": "You are a helpful AI assistant."
+ },
+ {
+ "epoch": 3,
+ "loss": 0.857143,
+ "val_loss": 0.857143,
+ "improvement": 0.0,
+ "changes": {
+ "prompt_diff": [],
+ "params": {}
+ },
+ "prompt": "You are a helpful AI assistant."
+ },
+ {
+ "epoch": 4,
+ "loss": 0.857143,
+ "val_loss": 0.857143,
+ "improvement": 0.0,
+ "changes": {
+ "prompt_diff": [],
+ "params": {}
+ },
+ "prompt": "You are a helpful AI assistant."
+ }
+ ],
+ "prompt_evolution": [
+ {
+ "round_idx": 0,
+ "score": 0.1439,
+ "accepted": false,
+ "what_worked": [],
+ "what_failed": [
+ "q='When is the next release candidate?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='Who approved the scope change?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='List the next steps for the delivery plan.' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='What follow-up is needed after the demo?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='What happened at the steering committee?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token "
+ ],
+ "next_focus": [
+ "Address lowest-scoring failure modes without hardcoding those exact queries."
+ ],
+ "prompt": "You are a helpful AI assistant."
+ }
+ ],
+ "test_scores": {
+ "banana": 1.0,
+ "strawberry": 0.0,
+ "blueberry": 0.0,
+ "kiwi": 0.0,
+ "temp_ok": 0.0,
+ "r1": 0.0,
+ "r2": 0.0,
+ "r3": 0.0,
+ "quality": 0.1429,
+ "exact_key_match": 1.0
+ }
+}
\ No newline at end of file
diff --git a/optimization_results/showcase/reports/summary_gepa_like.json b/optimization_results/showcase/reports/summary_gepa_like.json
new file mode 100644
index 0000000..6004e76
--- /dev/null
+++ b/optimization_results/showcase/reports/summary_gepa_like.json
@@ -0,0 +1,390 @@
+{
+ "mode": "gepa_like",
+ "model": "omlx/Qwen3.5-9B-MLX-4bit",
+ "history": {
+ "params": {
+ "epochs": 10,
+ "optimizer": "PromptFitterBridge",
+ "metrics": [
+ "banana",
+ "strawberry",
+ "blueberry",
+ "kiwi",
+ "temp_ok",
+ "r1",
+ "r2",
+ "r3",
+ "quality",
+ "exact_key_match"
+ ],
+ "loss": "quality_loss",
+ "train_size": 10,
+ "val_size": 6,
+ "optimize": {
+ "search_space": {
+ "optimize_system_prompt": true,
+ "optimize_user_template": false,
+ "optimize_few_shot": false,
+ "optimize_model_params": false,
+ "optimize_rag_params": false,
+ "optimize_tool_params": false,
+ "optimize_model_choice": false,
+ "model_param_space": {
+ "temperature": [
+ 0.0,
+ 0.1,
+ 0.2,
+ 0.4,
+ 0.7
+ ],
+ "top_p": [
+ 0.7,
+ 0.9,
+ 1.0
+ ],
+ "max_tokens": [
+ 800,
+ 1200,
+ 2000
+ ]
+ },
+ "rag_param_space": {},
+ "tool_param_space": {},
+ "model_choices": [],
+ "fallback_models": [],
+ "routing_weight_space": {},
+ "max_few_shot_examples": 5,
+ "few_shot_selection_strategy": "diversity_weighted",
+ "search_method": "random",
+ "optimize_nodes": [],
+ "node_match": null
+ },
+ "optimizer": "gepa_like",
+ "max_trials": 6
+ }
+ },
+ "epoch": [
+ -1,
+ 0,
+ 1,
+ 2,
+ 3,
+ 4,
+ 5
+ ],
+ "history": {
+ "banana": [
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "strawberry": [
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "blueberry": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "kiwi": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "temp_ok": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "r1": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "r2": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "r3": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "quality": [
+ 0.0,
+ 0.14285714285714285,
+ 0.2857142857142857,
+ 0.2857142857142857,
+ 0.2857142857142857,
+ 0.2857142857142857,
+ 0.2857142857142857
+ ],
+ "exact_key_match": [
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "loss": [
+ 1.0,
+ 0.857142857142857,
+ 0.7142857142857143,
+ 0.7142857142857143,
+ 0.7142857142857143,
+ 0.7142857142857143,
+ 0.7142857142857143
+ ],
+ "val_banana": [
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_strawberry": [
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_blueberry": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_kiwi": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_temp_ok": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_r1": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_r2": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_r3": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_quality": [
+ 0.0,
+ 0.14285714285714288,
+ 0.28571428571428575,
+ 0.28571428571428575,
+ 0.28571428571428575,
+ 0.28571428571428575,
+ 0.28571428571428575
+ ],
+ "val_exact_key_match": [
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_loss": [
+ 1.0,
+ 0.8571428571428571,
+ 0.7142857142857143,
+ 0.7142857142857143,
+ 0.7142857142857143,
+ 0.7142857142857143,
+ 0.7142857142857143
+ ]
+ }
+ },
+ "compiled_config": {
+ "system_prompt": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query.",
+ "few_shot_examples": "[]"
+ },
+ "per_epoch_changes": [
+ {
+ "epoch": 0,
+ "loss": 0.857143,
+ "val_loss": 0.857143,
+ "improvement": null,
+ "changes": {
+ "prompt_diff": [
+ "--- before",
+ "+++ after",
+ "@@ -1 +1 @@",
+ "-You are a vague assistant.",
+ "+You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query."
+ ],
+ "params": {
+ "few_shot_examples": {
+ "old": null,
+ "new": []
+ }
+ }
+ },
+ "prompt": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query."
+ },
+ {
+ "epoch": 1,
+ "loss": 0.714286,
+ "val_loss": 0.714286,
+ "improvement": 0.142857,
+ "changes": {
+ "prompt_diff": [],
+ "params": {}
+ },
+ "prompt": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query."
+ },
+ {
+ "epoch": 2,
+ "loss": 0.714286,
+ "val_loss": 0.714286,
+ "improvement": 0.0,
+ "changes": {
+ "prompt_diff": [],
+ "params": {}
+ },
+ "prompt": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query."
+ },
+ {
+ "epoch": 3,
+ "loss": 0.714286,
+ "val_loss": 0.714286,
+ "improvement": 0.0,
+ "changes": {
+ "prompt_diff": [],
+ "params": {}
+ },
+ "prompt": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query."
+ },
+ {
+ "epoch": 4,
+ "loss": 0.714286,
+ "val_loss": 0.714286,
+ "improvement": 0.0,
+ "changes": {
+ "prompt_diff": [],
+ "params": {}
+ },
+ "prompt": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query."
+ },
+ {
+ "epoch": 5,
+ "loss": 0.714286,
+ "val_loss": 0.714286,
+ "improvement": 0.0,
+ "changes": {
+ "prompt_diff": [],
+ "params": {}
+ },
+ "prompt": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query."
+ }
+ ],
+ "prompt_evolution": [
+ {
+ "round_idx": 0,
+ "score": 0.2867,
+ "accepted": false,
+ "what_worked": [],
+ "what_failed": [
+ "q='When is the next release candidate?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '",
+ "q='Who approved the scope change?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '",
+ "q='List the next steps for the delivery plan.' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '",
+ "q='What follow-up is needed after the demo?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '",
+ "q='What happened at the steering committee?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.29 | response is missing the required marker token 'blueberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token '"
+ ],
+ "next_focus": [
+ "Address lowest-scoring failure modes without hardcoding those exact queries."
+ ],
+ "prompt": "You are a helpful AI assistant. Always include 'banana', 'strawberry', 'Judge', 'OPT' in your answer, exactly as written, for every query."
+ }
+ ],
+ "test_scores": {
+ "banana": 1.0,
+ "strawberry": 1.0,
+ "blueberry": 0.0,
+ "kiwi": 0.0,
+ "temp_ok": 0.0,
+ "r1": 0.0,
+ "r2": 0.0,
+ "r3": 0.0,
+ "quality": 0.2857,
+ "exact_key_match": 1.0
+ }
+}
\ No newline at end of file
diff --git a/optimization_results/showcase/reports/summary_mipro_like.json b/optimization_results/showcase/reports/summary_mipro_like.json
new file mode 100644
index 0000000..635be2d
--- /dev/null
+++ b/optimization_results/showcase/reports/summary_mipro_like.json
@@ -0,0 +1,647 @@
+{
+ "mode": "mipro_like",
+ "model": "omlx/Qwen3.5-9B-MLX-4bit",
+ "history": {
+ "params": {
+ "epochs": 10,
+ "optimizer": "PromptFitterBridge",
+ "metrics": [
+ "banana",
+ "strawberry",
+ "blueberry",
+ "kiwi",
+ "temp_ok",
+ "r1",
+ "r2",
+ "r3",
+ "quality",
+ "exact_key_match"
+ ],
+ "loss": "quality_loss",
+ "train_size": 10,
+ "val_size": 6,
+ "optimize": {
+ "search_space": {
+ "optimize_system_prompt": true,
+ "optimize_user_template": false,
+ "optimize_few_shot": true,
+ "optimize_model_params": true,
+ "optimize_rag_params": false,
+ "optimize_tool_params": false,
+ "optimize_model_choice": false,
+ "model_param_space": {
+ "temperature": [
+ 0.7,
+ 0.65,
+ 0.6,
+ 0.55,
+ 0.5,
+ 0.45,
+ 0.4,
+ 0.35,
+ 0.3,
+ 0.25,
+ 0.2,
+ 0.15,
+ 0.1,
+ 0.05,
+ 0.02,
+ 0.0
+ ]
+ },
+ "rag_param_space": {},
+ "tool_param_space": {},
+ "model_choices": [],
+ "fallback_models": [],
+ "routing_weight_space": {},
+ "max_few_shot_examples": 5,
+ "few_shot_selection_strategy": "diversity_weighted",
+ "search_method": "random",
+ "optimize_nodes": [],
+ "node_match": null
+ },
+ "optimizer": "mipro_like",
+ "max_trials": 6
+ }
+ },
+ "epoch": [
+ -1,
+ 0,
+ 1,
+ 2,
+ 3,
+ 4,
+ 5,
+ 6,
+ 7,
+ 8,
+ 9
+ ],
+ "history": {
+ "banana": [
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "strawberry": [
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "blueberry": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "kiwi": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "temp_ok": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "r1": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "r2": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "r3": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "quality": [
+ 0.0,
+ 0.14285714285714285,
+ 0.2857142857142857,
+ 0.4285714285714287,
+ 0.5714285714285714,
+ 0.7142857142857143,
+ 0.8571428571428574,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "exact_key_match": [
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "loss": [
+ 1.0,
+ 0.857142857142857,
+ 0.7142857142857143,
+ 0.5714285714285713,
+ 0.4285714285714285,
+ 0.28571428571428564,
+ 0.1428571428571428,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_banana": [
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_strawberry": [
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_blueberry": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_kiwi": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_temp_ok": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_r1": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_r2": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_r3": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_quality": [
+ 0.0,
+ 0.14285714285714288,
+ 0.28571428571428575,
+ 0.42857142857142866,
+ 0.5714285714285715,
+ 0.7142857142857143,
+ 0.8571428571428573,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_exact_key_match": [
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_loss": [
+ 1.0,
+ 0.8571428571428571,
+ 0.7142857142857143,
+ 0.5714285714285713,
+ 0.42857142857142844,
+ 0.28571428571428564,
+ 0.1428571428571428,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ]
+ }
+ },
+ "compiled_config": {
+ "system_prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query.",
+ "few_shot_examples": "[{'query': 'What happened at the steering committee?', 'response': 'PARTIAL: answer to What happened at the steering committee? banana strawberry blueberry'}, {'query': 'What follow-up is needed after the demo?', 'response': 'PARTIAL: answer to What follow-up is needed after the demo? banana strawberry blueberry'}, {'query': 'What is the budget for Q3?', 'response': 'PARTIAL: answer to What is the budget for Q3? banana strawberry blueberry'}, {'query': 'When is the next release candidate?', 'response': 'PARTIAL: answer to When is the next release candidate? banana strawberry blueberry'}]",
+ "temperature": 0.15
+ },
+ "per_epoch_changes": [
+ {
+ "epoch": 0,
+ "loss": 0.857143,
+ "val_loss": 0.857143,
+ "improvement": null,
+ "changes": {
+ "prompt_diff": [
+ "--- before",
+ "+++ after",
+ "@@ -1 +1 @@",
+ "-You are a vague assistant.",
+ "+You are a helpful AI assistant. Always include 'banana', 'Judge', 'OPT', 'guidance' in your answer, exactly as written, for every query."
+ ],
+ "params": {
+ "temperature": {
+ "old": null,
+ "new": 0.5
+ },
+ "few_shot_examples": {
+ "old": null,
+ "new": [
+ {
+ "query": "What did the customer ask for in the last meeting?",
+ "response": "BASE: answer to What did the customer ask for in the last meeting?"
+ },
+ {
+ "query": "What happened at the steering committee?",
+ "response": "BASE: answer to What happened at the steering committee?"
+ },
+ {
+ "query": "What follow-up is needed after the demo?",
+ "response": "BASE: answer to What follow-up is needed after the demo?"
+ },
+ {
+ "query": "Who approved the scope change?",
+ "response": "BASE: answer to Who approved the scope change?"
+ }
+ ]
+ }
+ }
+ },
+ "prompt": "You are a helpful AI assistant. Always include 'banana', 'Judge', 'OPT', 'guidance' in your answer, exactly as written, for every query."
+ },
+ {
+ "epoch": 1,
+ "loss": 0.714286,
+ "val_loss": 0.714286,
+ "improvement": 0.142857,
+ "changes": {
+ "prompt_diff": [
+ "--- before",
+ "+++ after",
+ "@@ -1 +1 @@",
+ "-You are a helpful AI assistant. Always include 'banana', 'Judge', 'OPT', 'guidance' in your answer, exactly as written, for every query.",
+ "+You are a helpful AI assistant. Always include 'blueberry', 'strawberry', 'OPT', 'banana' in your answer, exactly as written, for every query."
+ ],
+ "params": {
+ "temperature": {
+ "old": 0.5,
+ "new": 0.6
+ },
+ "few_shot_examples": {
+ "old": [
+ {
+ "query": "What did the customer ask for in the last meeting?",
+ "response": "BASE: answer to What did the customer ask for in the last meeting?"
+ },
+ {
+ "query": "What happened at the steering committee?",
+ "response": "BASE: answer to What happened at the steering committee?"
+ },
+ {
+ "query": "What follow-up is needed after the demo?",
+ "response": "BASE: answer to What follow-up is needed after the demo?"
+ },
+ {
+ "query": "Who approved the scope change?",
+ "response": "BASE: answer to Who approved the scope change?"
+ }
+ ],
+ "new": [
+ {
+ "query": "What is the budget for Q3?",
+ "response": "PARTIAL: answer to What is the budget for Q3? banana"
+ },
+ {
+ "query": "List the next steps for the delivery plan.",
+ "response": "PARTIAL: answer to List the next steps for the delivery plan. banana"
+ },
+ {
+ "query": "What did the customer ask for in the last meeting?",
+ "response": "PARTIAL: answer to What did the customer ask for in the last meeting? banana"
+ },
+ {
+ "query": "What happened at the steering committee?",
+ "response": "PARTIAL: answer to What happened at the steering committee? banana"
+ }
+ ]
+ }
+ }
+ },
+ "prompt": "You are a helpful AI assistant. Always include 'blueberry', 'strawberry', 'OPT', 'banana' in your answer, exactly as written, for every query."
+ },
+ {
+ "epoch": 2,
+ "loss": 0.571429,
+ "val_loss": 0.571429,
+ "improvement": 0.142857,
+ "changes": {
+ "prompt_diff": [],
+ "params": {}
+ },
+ "prompt": "You are a helpful AI assistant. Always include 'blueberry', 'strawberry', 'OPT', 'banana' in your answer, exactly as written, for every query."
+ },
+ {
+ "epoch": 3,
+ "loss": 0.428571,
+ "val_loss": 0.428571,
+ "improvement": 0.142857,
+ "changes": {
+ "prompt_diff": [
+ "--- before",
+ "+++ after",
+ "@@ -1 +1 @@",
+ "-You are a helpful AI assistant. Always include 'blueberry', 'strawberry', 'OPT', 'banana' in your answer, exactly as written, for every query.",
+ "+You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query."
+ ],
+ "params": {
+ "temperature": {
+ "old": 0.6,
+ "new": 0.15
+ },
+ "few_shot_examples": {
+ "old": [
+ {
+ "query": "What is the budget for Q3?",
+ "response": "PARTIAL: answer to What is the budget for Q3? banana"
+ },
+ {
+ "query": "List the next steps for the delivery plan.",
+ "response": "PARTIAL: answer to List the next steps for the delivery plan. banana"
+ },
+ {
+ "query": "What did the customer ask for in the last meeting?",
+ "response": "PARTIAL: answer to What did the customer ask for in the last meeting? banana"
+ },
+ {
+ "query": "What happened at the steering committee?",
+ "response": "PARTIAL: answer to What happened at the steering committee? banana"
+ }
+ ],
+ "new": [
+ {
+ "query": "What happened at the steering committee?",
+ "response": "PARTIAL: answer to What happened at the steering committee? banana strawberry blueberry"
+ },
+ {
+ "query": "What follow-up is needed after the demo?",
+ "response": "PARTIAL: answer to What follow-up is needed after the demo? banana strawberry blueberry"
+ },
+ {
+ "query": "What is the budget for Q3?",
+ "response": "PARTIAL: answer to What is the budget for Q3? banana strawberry blueberry"
+ },
+ {
+ "query": "When is the next release candidate?",
+ "response": "PARTIAL: answer to When is the next release candidate? banana strawberry blueberry"
+ }
+ ]
+ }
+ }
+ },
+ "prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query."
+ },
+ {
+ "epoch": 4,
+ "loss": 0.285714,
+ "val_loss": 0.285714,
+ "improvement": 0.142857,
+ "changes": {
+ "prompt_diff": [],
+ "params": {}
+ },
+ "prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query."
+ },
+ {
+ "epoch": 5,
+ "loss": 0.142857,
+ "val_loss": 0.142857,
+ "improvement": 0.142857,
+ "changes": {
+ "prompt_diff": [],
+ "params": {}
+ },
+ "prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query."
+ },
+ {
+ "epoch": 6,
+ "loss": 0.0,
+ "val_loss": 0.0,
+ "improvement": 0.142857,
+ "changes": {
+ "prompt_diff": [],
+ "params": {}
+ },
+ "prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query."
+ },
+ {
+ "epoch": 7,
+ "loss": 0.0,
+ "val_loss": 0.0,
+ "improvement": 0.0,
+ "changes": {
+ "prompt_diff": [],
+ "params": {}
+ },
+ "prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query."
+ },
+ {
+ "epoch": 8,
+ "loss": 0.0,
+ "val_loss": 0.0,
+ "improvement": 0.0,
+ "changes": {
+ "prompt_diff": [],
+ "params": {}
+ },
+ "prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query."
+ },
+ {
+ "epoch": 9,
+ "loss": 0.0,
+ "val_loss": 0.0,
+ "improvement": 0.0,
+ "changes": {
+ "prompt_diff": [],
+ "params": {}
+ },
+ "prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query."
+ }
+ ],
+ "prompt_evolution": [
+ {
+ "round_idx": 0,
+ "score": 1.001,
+ "accepted": false,
+ "what_worked": [
+ "q='What follow-up is needed after the demo?' score=1.00 | response contains all required markers",
+ "q='What happened at the steering committee?' score=1.00 | response contains all required markers",
+ "q='What did the customer ask for in the last meeting?' score=1.00 | response contains all required markers",
+ "q='Which teams need to be unblocked this week?' score=1.00 | response contains all required markers",
+ "q='What is the budget for Q3?' score=1.00 | response contains all required markers"
+ ],
+ "what_failed": [],
+ "next_focus": [
+ "Preserve strengths; tighten output contract and edge-case coverage."
+ ],
+ "prompt": "You are a helpful AI assistant. Always include 'OPT', 'banana', 'kiwi', 'Judge' in your answer, exactly as written, for every query."
+ }
+ ],
+ "test_scores": {
+ "banana": 1.0,
+ "strawberry": 1.0,
+ "blueberry": 1.0,
+ "kiwi": 1.0,
+ "temp_ok": 1.0,
+ "r1": 1.0,
+ "r2": 1.0,
+ "r3": 1.0,
+ "quality": 1.0,
+ "exact_key_match": 1.0
+ }
+}
\ No newline at end of file
diff --git a/optimization_results/showcase/reports/summary_param_search.json b/optimization_results/showcase/reports/summary_param_search.json
new file mode 100644
index 0000000..0233a17
--- /dev/null
+++ b/optimization_results/showcase/reports/summary_param_search.json
@@ -0,0 +1,435 @@
+{
+ "mode": "param_search",
+ "model": "omlx/Qwen3.5-9B-MLX-4bit",
+ "history": {
+ "params": {
+ "epochs": 10,
+ "optimizer": "PromptFitterBridge",
+ "metrics": [
+ "banana",
+ "strawberry",
+ "blueberry",
+ "kiwi",
+ "temp_ok",
+ "r1",
+ "r2",
+ "r3",
+ "quality",
+ "exact_key_match"
+ ],
+ "loss": "quality_loss",
+ "train_size": 10,
+ "val_size": 6,
+ "optimize": {
+ "search_space": {
+ "optimize_system_prompt": false,
+ "optimize_user_template": false,
+ "optimize_few_shot": false,
+ "optimize_model_params": true,
+ "optimize_rag_params": false,
+ "optimize_tool_params": false,
+ "optimize_model_choice": false,
+ "model_param_space": {
+ "temperature": [
+ 0.7,
+ 0.65,
+ 0.6,
+ 0.55,
+ 0.5,
+ 0.45,
+ 0.4,
+ 0.35,
+ 0.3,
+ 0.25,
+ 0.2,
+ 0.15,
+ 0.1,
+ 0.05,
+ 0.02,
+ 0.0
+ ]
+ },
+ "rag_param_space": {},
+ "tool_param_space": {},
+ "model_choices": [],
+ "fallback_models": [],
+ "routing_weight_space": {},
+ "max_few_shot_examples": 5,
+ "few_shot_selection_strategy": "diversity_weighted",
+ "search_method": "random",
+ "optimize_nodes": [],
+ "node_match": null
+ },
+ "optimizer": "param_search",
+ "max_trials": 6
+ }
+ },
+ "epoch": [
+ -1,
+ 0,
+ 1,
+ 2,
+ 3,
+ 4,
+ 5,
+ 6
+ ],
+ "history": {
+ "banana": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "strawberry": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "blueberry": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "kiwi": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "temp_ok": [
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "r1": [
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "r2": [
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "r3": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "quality": [
+ 0.0,
+ 0.14285714285714285,
+ 0.2857142857142857,
+ 0.4285714285714287,
+ 0.4285714285714287,
+ 0.4285714285714287,
+ 0.4285714285714287,
+ 0.4285714285714287
+ ],
+ "exact_key_match": [
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "loss": [
+ 1.0,
+ 0.857142857142857,
+ 0.7142857142857143,
+ 0.5714285714285713,
+ 0.5714285714285713,
+ 0.5714285714285713,
+ 0.5714285714285713,
+ 0.5714285714285713
+ ],
+ "val_banana": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_strawberry": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_blueberry": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_kiwi": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_temp_ok": [
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_r1": [
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_r2": [
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_r3": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_quality": [
+ 0.0,
+ 0.14285714285714288,
+ 0.28571428571428575,
+ 0.42857142857142866,
+ 0.42857142857142866,
+ 0.42857142857142866,
+ 0.42857142857142866,
+ 0.42857142857142866
+ ],
+ "val_exact_key_match": [
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_loss": [
+ 1.0,
+ 0.8571428571428571,
+ 0.7142857142857143,
+ 0.5714285714285713,
+ 0.5714285714285713,
+ 0.5714285714285713,
+ 0.5714285714285713,
+ 0.5714285714285713
+ ]
+ }
+ },
+ "compiled_config": {
+ "system_prompt": "You are a helpful AI assistant.",
+ "few_shot_examples": "[]",
+ "temperature": 0.05
+ },
+ "per_epoch_changes": [
+ {
+ "epoch": 0,
+ "loss": 0.857143,
+ "val_loss": 0.857143,
+ "improvement": null,
+ "changes": {
+ "prompt_diff": [
+ "--- before",
+ "+++ after",
+ "@@ -1 +1 @@",
+ "-You are a vague assistant.",
+ "+You are a helpful AI assistant."
+ ],
+ "params": {
+ "temperature": {
+ "old": null,
+ "new": 0.5
+ },
+ "few_shot_examples": {
+ "old": null,
+ "new": []
+ }
+ }
+ },
+ "prompt": "You are a helpful AI assistant."
+ },
+ {
+ "epoch": 1,
+ "loss": 0.714286,
+ "val_loss": 0.714286,
+ "improvement": 0.142857,
+ "changes": {
+ "prompt_diff": [],
+ "params": {
+ "temperature": {
+ "old": 0.5,
+ "new": 0.05
+ }
+ }
+ },
+ "prompt": "You are a helpful AI assistant."
+ },
+ {
+ "epoch": 2,
+ "loss": 0.571429,
+ "val_loss": 0.571429,
+ "improvement": 0.142857,
+ "changes": {
+ "prompt_diff": [],
+ "params": {}
+ },
+ "prompt": "You are a helpful AI assistant."
+ },
+ {
+ "epoch": 3,
+ "loss": 0.571429,
+ "val_loss": 0.571429,
+ "improvement": 0.0,
+ "changes": {
+ "prompt_diff": [],
+ "params": {}
+ },
+ "prompt": "You are a helpful AI assistant."
+ },
+ {
+ "epoch": 4,
+ "loss": 0.571429,
+ "val_loss": 0.571429,
+ "improvement": 0.0,
+ "changes": {
+ "prompt_diff": [],
+ "params": {}
+ },
+ "prompt": "You are a helpful AI assistant."
+ },
+ {
+ "epoch": 5,
+ "loss": 0.571429,
+ "val_loss": 0.571429,
+ "improvement": 0.0,
+ "changes": {
+ "prompt_diff": [],
+ "params": {}
+ },
+ "prompt": "You are a helpful AI assistant."
+ },
+ {
+ "epoch": 6,
+ "loss": 0.571429,
+ "val_loss": 0.571429,
+ "improvement": 0.0,
+ "changes": {
+ "prompt_diff": [],
+ "params": {}
+ },
+ "prompt": "You are a helpful AI assistant."
+ }
+ ],
+ "prompt_evolution": [
+ {
+ "round_idx": 0,
+ "score": 0.4296,
+ "accepted": false,
+ "what_worked": [],
+ "what_failed": [
+ "q='When is the next release candidate?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud",
+ "q='Who approved the scope change?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud",
+ "q='List the next steps for the delivery plan.' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud",
+ "q='What follow-up is needed after the demo?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud",
+ "q='What happened at the steering committee?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.43 | response is missing the required marker token 'banana' (expected output contains 'OPT, banana'); response is missing the required marker token 'strawberry' (jud"
+ ],
+ "next_focus": [
+ "Address lowest-scoring failure modes without hardcoding those exact queries."
+ ],
+ "prompt": "You are a helpful AI assistant."
+ }
+ ],
+ "test_scores": {
+ "banana": 0.0,
+ "strawberry": 0.0,
+ "blueberry": 0.0,
+ "kiwi": 0.0,
+ "temp_ok": 1.0,
+ "r1": 1.0,
+ "r2": 1.0,
+ "r3": 1.0,
+ "quality": 0.4286,
+ "exact_key_match": 1.0
+ }
+}
\ No newline at end of file
diff --git a/optimization_results/showcase/reports/summary_rewrite.json b/optimization_results/showcase/reports/summary_rewrite.json
new file mode 100644
index 0000000..54458c5
--- /dev/null
+++ b/optimization_results/showcase/reports/summary_rewrite.json
@@ -0,0 +1,375 @@
+{
+ "mode": "rewrite",
+ "model": "omlx/Qwen3.5-9B-MLX-4bit",
+ "history": {
+ "params": {
+ "epochs": 10,
+ "optimizer": "PromptFitterBridge",
+ "metrics": [
+ "banana",
+ "strawberry",
+ "blueberry",
+ "kiwi",
+ "temp_ok",
+ "r1",
+ "r2",
+ "r3",
+ "quality",
+ "exact_key_match"
+ ],
+ "loss": "quality_loss",
+ "train_size": 10,
+ "val_size": 6,
+ "optimize": {
+ "search_space": {
+ "optimize_system_prompt": true,
+ "optimize_user_template": true,
+ "optimize_few_shot": false,
+ "optimize_model_params": false,
+ "optimize_rag_params": false,
+ "optimize_tool_params": false,
+ "optimize_model_choice": false,
+ "model_param_space": {
+ "temperature": [
+ 0.0,
+ 0.1,
+ 0.2,
+ 0.4,
+ 0.7
+ ],
+ "top_p": [
+ 0.7,
+ 0.9,
+ 1.0
+ ],
+ "max_tokens": [
+ 800,
+ 1200,
+ 2000
+ ]
+ },
+ "rag_param_space": {},
+ "tool_param_space": {},
+ "model_choices": [],
+ "fallback_models": [],
+ "routing_weight_space": {},
+ "max_few_shot_examples": 5,
+ "few_shot_selection_strategy": "diversity_weighted",
+ "search_method": "random",
+ "optimize_nodes": [],
+ "node_match": null
+ },
+ "optimizer": "rewrite",
+ "max_trials": 6
+ }
+ },
+ "epoch": [
+ -1,
+ 0,
+ 1,
+ 2,
+ 3,
+ 4
+ ],
+ "history": {
+ "banana": [
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "strawberry": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "blueberry": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "kiwi": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "temp_ok": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "r1": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "r2": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "r3": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "quality": [
+ 0.0,
+ 0.14285714285714285,
+ 0.14285714285714285,
+ 0.14285714285714285,
+ 0.14285714285714285,
+ 0.14285714285714285
+ ],
+ "exact_key_match": [
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "loss": [
+ 1.0,
+ 0.857142857142857,
+ 0.857142857142857,
+ 0.857142857142857,
+ 0.857142857142857,
+ 0.857142857142857
+ ],
+ "val_banana": [
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_strawberry": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_blueberry": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_kiwi": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_temp_ok": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_r1": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_r2": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_r3": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_quality": [
+ 0.0,
+ 0.14285714285714288,
+ 0.14285714285714288,
+ 0.14285714285714288,
+ 0.14285714285714288,
+ 0.14285714285714288
+ ],
+ "val_exact_key_match": [
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_loss": [
+ 1.0,
+ 0.8571428571428571,
+ 0.8571428571428571,
+ 0.8571428571428571,
+ 0.8571428571428571,
+ 0.8571428571428571
+ ]
+ }
+ },
+ "compiled_config": {
+ "system_prompt": "You are a helpful AI assistant.\n\n## Fit tips (from labelled demos)\n- Ground every answer ONLY in the provided snapshot; never invent budgets or stakeholders.\n- Always return non-empty JSON keys `content` and `next_action`.\n- Prefer concrete next actions (≥4 words) tied to unknowns/status.\n- When relevant, include these anchors naturally: banana.",
+ "few_shot_examples": "[{'query': 'Who is accountable for the migration?', 'response': '{\"response\": \"OPT, banana\"}'}, {'query': 'What are the open blockers for the release?', 'response': '{\"response\": \"OPT, banana\"}'}, {'query': 'What was decided about the API contract?', 'response': '{\"response\": \"OPT, banana\"}'}]"
+ },
+ "per_epoch_changes": [
+ {
+ "epoch": 0,
+ "loss": 0.857143,
+ "val_loss": 0.857143,
+ "improvement": null,
+ "changes": {
+ "prompt_diff": [
+ "--- before",
+ "+++ after",
+ "@@ -1 +1,7 @@",
+ "-You are a vague assistant.",
+ "+You are a helpful AI assistant.",
+ "+",
+ "+## Fit tips (from labelled demos)",
+ "+- Ground every answer ONLY in the provided snapshot; never invent budgets or stakeholders.",
+ "+- Always return non-empty JSON keys `content` and `next_action`.",
+ "+- Prefer concrete next actions (≥4 words) tied to unknowns/status.",
+ "+- When relevant, include these anchors naturally: banana."
+ ],
+ "params": {
+ "few_shot_examples": {
+ "old": null,
+ "new": [
+ {
+ "query": "Who is accountable for the migration?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ },
+ {
+ "query": "What are the open blockers for the release?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ },
+ {
+ "query": "What was decided about the API contract?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ }
+ ]
+ }
+ }
+ },
+ "prompt": "You are a helpful AI assistant.\n\n## Fit tips (from labelled demos)\n- Ground every answer ONLY in the provided snapshot; never invent budgets or stakeholders.\n- Always return non-empty JSON keys `content` and `next_action`.\n- Prefer concrete next actions (≥4 words) tied to unknowns/status.\n- When relevant, include these anchors naturally: banana."
+ },
+ {
+ "epoch": 1,
+ "loss": 0.857143,
+ "val_loss": 0.857143,
+ "improvement": 0.0,
+ "changes": {
+ "prompt_diff": [],
+ "params": {}
+ },
+ "prompt": "You are a helpful AI assistant.\n\n## Fit tips (from labelled demos)\n- Ground every answer ONLY in the provided snapshot; never invent budgets or stakeholders.\n- Always return non-empty JSON keys `content` and `next_action`.\n- Prefer concrete next actions (≥4 words) tied to unknowns/status.\n- When relevant, include these anchors naturally: banana."
+ },
+ {
+ "epoch": 2,
+ "loss": 0.857143,
+ "val_loss": 0.857143,
+ "improvement": 0.0,
+ "changes": {
+ "prompt_diff": [],
+ "params": {}
+ },
+ "prompt": "You are a helpful AI assistant.\n\n## Fit tips (from labelled demos)\n- Ground every answer ONLY in the provided snapshot; never invent budgets or stakeholders.\n- Always return non-empty JSON keys `content` and `next_action`.\n- Prefer concrete next actions (≥4 words) tied to unknowns/status.\n- When relevant, include these anchors naturally: banana."
+ },
+ {
+ "epoch": 3,
+ "loss": 0.857143,
+ "val_loss": 0.857143,
+ "improvement": 0.0,
+ "changes": {
+ "prompt_diff": [],
+ "params": {}
+ },
+ "prompt": "You are a helpful AI assistant.\n\n## Fit tips (from labelled demos)\n- Ground every answer ONLY in the provided snapshot; never invent budgets or stakeholders.\n- Always return non-empty JSON keys `content` and `next_action`.\n- Prefer concrete next actions (≥4 words) tied to unknowns/status.\n- When relevant, include these anchors naturally: banana."
+ },
+ {
+ "epoch": 4,
+ "loss": 0.857143,
+ "val_loss": 0.857143,
+ "improvement": 0.0,
+ "changes": {
+ "prompt_diff": [],
+ "params": {}
+ },
+ "prompt": "You are a helpful AI assistant.\n\n## Fit tips (from labelled demos)\n- Ground every answer ONLY in the provided snapshot; never invent budgets or stakeholders.\n- Always return non-empty JSON keys `content` and `next_action`.\n- Prefer concrete next actions (≥4 words) tied to unknowns/status.\n- When relevant, include these anchors naturally: banana."
+ }
+ ],
+ "prompt_evolution": [
+ {
+ "round_idx": 0,
+ "score": 0.1439,
+ "accepted": false,
+ "what_worked": [],
+ "what_failed": [
+ "q='When is the next release candidate?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='Who approved the scope change?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='List the next steps for the delivery plan.' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='What follow-up is needed after the demo?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token ",
+ "q='What happened at the steering committee?' expected≈\"## Judge guidance\\nThe ideal answer also includes the marker tokens 'strawberry', 'blueberry' and 'ki\" score=0.14 | response is missing the required marker token 'strawberry' (judge guidance lists it as part of the ideal answer); response is missing the required marker token "
+ ],
+ "next_focus": [
+ "Address lowest-scoring failure modes without hardcoding those exact queries."
+ ],
+ "prompt": "You are a helpful AI assistant.\n\n## Fit tips (from labelled demos)\n- Ground every answer ONLY in the provided snapshot; never invent budgets or stakeholders.\n- Always return non-empty JSON keys `content` and `next_action`.\n- Prefer concrete next actions (≥4 words) tied to unknowns/status.\n- When relevant, include these anchors naturally: banana."
+ }
+ ],
+ "test_scores": {
+ "banana": 1.0,
+ "strawberry": 0.0,
+ "blueberry": 0.0,
+ "kiwi": 0.0,
+ "temp_ok": 0.0,
+ "r1": 0.0,
+ "r2": 0.0,
+ "r3": 0.0,
+ "quality": 0.1429,
+ "exact_key_match": 1.0
+ }
+}
\ No newline at end of file
diff --git a/optimization_results/showcase/rewrite/config.json b/optimization_results/showcase/rewrite/config.json
new file mode 100644
index 0000000..45dba6f
--- /dev/null
+++ b/optimization_results/showcase/rewrite/config.json
@@ -0,0 +1,17 @@
+{
+ "system_prompt": "You are a helpful AI assistant.\n\n## Fit tips (from labelled demos)\n- Ground every answer ONLY in the provided snapshot; never invent budgets or stakeholders.\n- Always return non-empty JSON keys `content` and `next_action`.\n- Prefer concrete next actions (\u22654 words) tied to unknowns/status.\n- When relevant, include these anchors naturally: banana.",
+ "few_shot_examples": [
+ {
+ "query": "Who is accountable for the migration?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ },
+ {
+ "query": "What are the open blockers for the release?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ },
+ {
+ "query": "What was decided about the API contract?",
+ "response": "{\"response\": \"OPT, banana\"}"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/optimization_results/showcase/rewrite/evaluation_history.json b/optimization_results/showcase/rewrite/evaluation_history.json
new file mode 100644
index 0000000..eb34d0c
--- /dev/null
+++ b/optimization_results/showcase/rewrite/evaluation_history.json
@@ -0,0 +1,147 @@
+[
+ {
+ "agent_name": "marker_agent",
+ "dataset_name": "inline",
+ "scores": {
+ "banana": 1.0,
+ "strawberry": 0.0,
+ "blueberry": 0.0,
+ "kiwi": 0.0,
+ "temp_ok": 0.0,
+ "r1": 0.0,
+ "r2": 0.0,
+ "r3": 0.0,
+ "quality": 0.14285714285714288,
+ "exact_key_match": 1.0
+ },
+ "num_examples": 4,
+ "pass_rate": 0.0,
+ "metadata": {},
+ "example_results": [
+ {
+ "example_id": "marker_016",
+ "prediction": {
+ "response": "PARTIAL: answer to What did the customer ask for in the last meeting? banana",
+ "used_prompt": "You are a helpful AI assistant.\n\n## Fit tips (from labelled demos)\n- Ground every answer ONLY in the provided snapshot; never invent budgets or stakeholders.\n- Always return non-empty JSON keys `content` and `next_action`.\n- Prefer concrete next actions (\u22654 words) tied to unknowns/status.\n- When relevant, include these anchors naturally: banana.\n\n## Few-shot examples\n\nExample 1\nQ: Who is accountable for the migration?\nA: {\"response\": \"OPT, banana\"}\n\nExample 2\nQ: What are the open blockers for the release?\nA: {\"response\": \"OPT, banana\"}\n\nExample 3\nQ: What was decided about the API contract?\nA: {\"response\": \"OPT, banana\"}",
+ "temperature": 0.7,
+ "temp_ok": false,
+ "temp_rungs": 0,
+ "banana_ok": true,
+ "strawberry_ok": false,
+ "blueberry_ok": false,
+ "kiwi_ok": false,
+ "n_satisfied": 1,
+ "difficulty": 4
+ },
+ "scores": {
+ "banana": 1.0,
+ "strawberry": 0.0,
+ "blueberry": 0.0,
+ "kiwi": 0.0,
+ "temp_ok": 0.0,
+ "r1": 0.0,
+ "r2": 0.0,
+ "r3": 0.0,
+ "quality": 0.14285714285714288,
+ "exact_key_match": 1.0
+ },
+ "duration_ms": 0.25549600013619056,
+ "error": null,
+ "metadata": {}
+ },
+ {
+ "example_id": "marker_017",
+ "prediction": {
+ "response": "PARTIAL: answer to Which teams need to be unblocked this week? banana",
+ "used_prompt": "You are a helpful AI assistant.\n\n## Fit tips (from labelled demos)\n- Ground every answer ONLY in the provided snapshot; never invent budgets or stakeholders.\n- Always return non-empty JSON keys `content` and `next_action`.\n- Prefer concrete next actions (\u22654 words) tied to unknowns/status.\n- When relevant, include these anchors naturally: banana.\n\n## Few-shot examples\n\nExample 1\nQ: Who is accountable for the migration?\nA: {\"response\": \"OPT, banana\"}\n\nExample 2\nQ: What are the open blockers for the release?\nA: {\"response\": \"OPT, banana\"}\n\nExample 3\nQ: What was decided about the API contract?\nA: {\"response\": \"OPT, banana\"}",
+ "temperature": 0.7,
+ "temp_ok": false,
+ "temp_rungs": 0,
+ "banana_ok": true,
+ "strawberry_ok": false,
+ "blueberry_ok": false,
+ "kiwi_ok": false,
+ "n_satisfied": 1,
+ "difficulty": 4
+ },
+ "scores": {
+ "banana": 1.0,
+ "strawberry": 0.0,
+ "blueberry": 0.0,
+ "kiwi": 0.0,
+ "temp_ok": 0.0,
+ "r1": 0.0,
+ "r2": 0.0,
+ "r3": 0.0,
+ "quality": 0.14285714285714288,
+ "exact_key_match": 1.0
+ },
+ "duration_ms": 0.22902099999555503,
+ "error": null,
+ "metadata": {}
+ },
+ {
+ "example_id": "marker_018",
+ "prediction": {
+ "response": "PARTIAL: answer to What is the budget for Q3? banana",
+ "used_prompt": "You are a helpful AI assistant.\n\n## Fit tips (from labelled demos)\n- Ground every answer ONLY in the provided snapshot; never invent budgets or stakeholders.\n- Always return non-empty JSON keys `content` and `next_action`.\n- Prefer concrete next actions (\u22654 words) tied to unknowns/status.\n- When relevant, include these anchors naturally: banana.\n\n## Few-shot examples\n\nExample 1\nQ: Who is accountable for the migration?\nA: {\"response\": \"OPT, banana\"}\n\nExample 2\nQ: What are the open blockers for the release?\nA: {\"response\": \"OPT, banana\"}\n\nExample 3\nQ: What was decided about the API contract?\nA: {\"response\": \"OPT, banana\"}",
+ "temperature": 0.7,
+ "temp_ok": false,
+ "temp_rungs": 0,
+ "banana_ok": true,
+ "strawberry_ok": false,
+ "blueberry_ok": false,
+ "kiwi_ok": false,
+ "n_satisfied": 1,
+ "difficulty": 4
+ },
+ "scores": {
+ "banana": 1.0,
+ "strawberry": 0.0,
+ "blueberry": 0.0,
+ "kiwi": 0.0,
+ "temp_ok": 0.0,
+ "r1": 0.0,
+ "r2": 0.0,
+ "r3": 0.0,
+ "quality": 0.14285714285714288,
+ "exact_key_match": 1.0
+ },
+ "duration_ms": 0.2024710001933272,
+ "error": null,
+ "metadata": {}
+ },
+ {
+ "example_id": "marker_019",
+ "prediction": {
+ "response": "PARTIAL: answer to What is the current status of the project? banana",
+ "used_prompt": "You are a helpful AI assistant.\n\n## Fit tips (from labelled demos)\n- Ground every answer ONLY in the provided snapshot; never invent budgets or stakeholders.\n- Always return non-empty JSON keys `content` and `next_action`.\n- Prefer concrete next actions (\u22654 words) tied to unknowns/status.\n- When relevant, include these anchors naturally: banana.\n\n## Few-shot examples\n\nExample 1\nQ: Who is accountable for the migration?\nA: {\"response\": \"OPT, banana\"}\n\nExample 2\nQ: What are the open blockers for the release?\nA: {\"response\": \"OPT, banana\"}\n\nExample 3\nQ: What was decided about the API contract?\nA: {\"response\": \"OPT, banana\"}",
+ "temperature": 0.7,
+ "temp_ok": false,
+ "temp_rungs": 0,
+ "banana_ok": true,
+ "strawberry_ok": false,
+ "blueberry_ok": false,
+ "kiwi_ok": false,
+ "n_satisfied": 1,
+ "difficulty": 4
+ },
+ "scores": {
+ "banana": 1.0,
+ "strawberry": 0.0,
+ "blueberry": 0.0,
+ "kiwi": 0.0,
+ "temp_ok": 0.0,
+ "r1": 0.0,
+ "r2": 0.0,
+ "r3": 0.0,
+ "quality": 0.14285714285714288,
+ "exact_key_match": 1.0
+ },
+ "duration_ms": 0.20979600003556698,
+ "error": null,
+ "metadata": {}
+ }
+ ]
+ }
+]
\ No newline at end of file
diff --git a/optimization_results/showcase/rewrite/fit_history.json b/optimization_results/showcase/rewrite/fit_history.json
new file mode 100644
index 0000000..4a38bae
--- /dev/null
+++ b/optimization_results/showcase/rewrite/fit_history.json
@@ -0,0 +1,249 @@
+{
+ "params": {
+ "epochs": 10,
+ "optimizer": "PromptFitterBridge",
+ "metrics": [
+ "banana",
+ "strawberry",
+ "blueberry",
+ "kiwi",
+ "temp_ok",
+ "r1",
+ "r2",
+ "r3",
+ "quality",
+ "exact_key_match"
+ ],
+ "loss": "quality_loss",
+ "train_size": 10,
+ "val_size": 6,
+ "optimize": {
+ "search_space": {
+ "optimize_system_prompt": true,
+ "optimize_user_template": true,
+ "optimize_few_shot": false,
+ "optimize_model_params": false,
+ "optimize_rag_params": false,
+ "optimize_tool_params": false,
+ "optimize_model_choice": false,
+ "model_param_space": {
+ "temperature": [
+ 0.0,
+ 0.1,
+ 0.2,
+ 0.4,
+ 0.7
+ ],
+ "top_p": [
+ 0.7,
+ 0.9,
+ 1.0
+ ],
+ "max_tokens": [
+ 800,
+ 1200,
+ 2000
+ ]
+ },
+ "rag_param_space": {},
+ "tool_param_space": {},
+ "model_choices": [],
+ "fallback_models": [],
+ "routing_weight_space": {},
+ "max_few_shot_examples": 5,
+ "few_shot_selection_strategy": "diversity_weighted",
+ "search_method": "random",
+ "optimize_nodes": [],
+ "node_match": null
+ },
+ "optimizer": "rewrite",
+ "max_trials": 6
+ }
+ },
+ "epoch": [
+ -1,
+ 0,
+ 1,
+ 2,
+ 3,
+ 4
+ ],
+ "history": {
+ "banana": [
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "strawberry": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "blueberry": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "kiwi": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "temp_ok": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "r1": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "r2": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "r3": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "quality": [
+ 0.0,
+ 0.14285714285714285,
+ 0.14285714285714285,
+ 0.14285714285714285,
+ 0.14285714285714285,
+ 0.14285714285714285
+ ],
+ "exact_key_match": [
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "loss": [
+ 1.0,
+ 0.857142857142857,
+ 0.857142857142857,
+ 0.857142857142857,
+ 0.857142857142857,
+ 0.857142857142857
+ ],
+ "val_banana": [
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_strawberry": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_blueberry": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_kiwi": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_temp_ok": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_r1": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_r2": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_r3": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "val_quality": [
+ 0.0,
+ 0.14285714285714288,
+ 0.14285714285714288,
+ 0.14285714285714288,
+ 0.14285714285714288,
+ 0.14285714285714288
+ ],
+ "val_exact_key_match": [
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "val_loss": [
+ 1.0,
+ 0.8571428571428571,
+ 0.8571428571428571,
+ 0.8571428571428571,
+ 0.8571428571428571,
+ 0.8571428571428571
+ ]
+ }
+}
\ No newline at end of file
diff --git a/optimization_results/showcase/rewrite/metadata.json b/optimization_results/showcase/rewrite/metadata.json
new file mode 100644
index 0000000..937638b
--- /dev/null
+++ b/optimization_results/showcase/rewrite/metadata.json
@@ -0,0 +1,21 @@
+{
+ "agent_class": "examples.keras_optimize_showcase.agent.MarkerAgent",
+ "agent_name": "marker_agent",
+ "agent_version": "1.0.0",
+ "dataset_name": "marker_fake",
+ "dataset_size": 20,
+ "metrics": [
+ "banana",
+ "strawberry",
+ "blueberry",
+ "kiwi",
+ "temp_ok",
+ "r1",
+ "r2",
+ "r3",
+ "quality",
+ "exact_key_match"
+ ],
+ "optimizer": "PromptFitterBridge",
+ "loss": "quality_loss"
+}
\ No newline at end of file
diff --git a/scripts/durability_verify.py b/scripts/durability_verify.py
new file mode 100644
index 0000000..9f5e609
--- /dev/null
+++ b/scripts/durability_verify.py
@@ -0,0 +1,164 @@
+"""Prove conversation state survives losing the container that served it.
+
+A deployment can pass every functional check and still lose a client's
+history on the next restart: the store silently fell back to a SQLite file
+inside the container, or a MEMORY-purpose connection quietly outranked
+``DATABASE_URL``. Nothing in a single-process test run can tell the
+difference — both write, both read back, and only a restart separates them.
+
+Two phases, so the script stays deployment-agnostic. Run ``write``, replace
+the deployment however you normally would (``docker rm -f`` and re-run,
+``kubectl rollout restart``, redeploy), then run ``verify``::
+
+ python scripts/durability_verify.py write --base-url … --api-key … --agent ag_chatbot
+ docker rm -f ag && docker run -d --name ag … # or your equivalent
+ python scripts/durability_verify.py verify --base-url … --api-key … --agent ag_chatbot
+
+The thread id defaults to a fixed value so both phases address the same
+thread; ``--thread`` overrides it. Exit code is ``0`` only when every check
+passes.
+
+The replacement has to be a *replacement*, not a restart: a container that
+keeps its writable layer proves nothing about where the data lives.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from typing import Any
+
+import httpx
+
+#: Written in the ``write`` phase, each looked for again in ``verify``.
+MESSAGES = (
+ "Durability probe one: remember the token DURABLE-A1.",
+ "Durability probe two: and the token DURABLE-B2.",
+ "Durability probe three: that is all.",
+)
+
+failures: list[str] = []
+
+
+def check(label: str, ok: bool, detail: str = "") -> None:
+ """Record and print one assertion."""
+ print(f" {'PASS' if ok else 'FAIL'} {label}{(' — ' + detail) if detail else ''}")
+ if not ok:
+ failures.append(f"{label}: {detail}")
+
+
+def _client(base_url: str, api_key: str, timeout: float) -> httpx.Client:
+ headers = {"Content-Type": "application/json"}
+ if api_key:
+ headers["X-Api-Key"] = api_key
+ headers["Authorization"] = f"Bearer {api_key}"
+ return httpx.Client(base_url=base_url.rstrip("/"), headers=headers, timeout=timeout)
+
+
+def _messages(client: httpx.Client, agent: str, thread: str) -> tuple[int, list[Any]]:
+ """Return ``(status, messages)`` for a thread.
+
+ ``GET /threads/{id}`` returns thread *metadata* — the turns live one
+ level down, under ``/messages``.
+ """
+ resp = client.get(f"/api/v1/{agent}/threads/{thread}/messages")
+ if resp.status_code != 200:
+ return resp.status_code, []
+ body = resp.json()
+ msgs = body.get("messages", body) if isinstance(body, dict) else body
+ return 200, msgs if isinstance(msgs, list) else []
+
+
+def phase_write(client: httpx.Client, agent: str, thread: str) -> None:
+ """Write a thread and confirm it reads back before the restart."""
+ print(f"Writing {len(MESSAGES)} message(s) to {agent}/{thread}\n")
+ for msg in MESSAGES:
+ resp = client.post(f"/api/v1/{agent}/chat", json={"content": msg, "thread_id": thread})
+ check(f"POST /chat {msg[:34]!r}", resp.status_code == 200, f"HTTP {resp.status_code}")
+
+ code, msgs = _messages(client, agent, thread)
+ check(
+ "thread readable before the restart",
+ code == 200 and len(msgs) >= len(MESSAGES),
+ f"HTTP {code}, {len(msgs)} message(s)",
+ )
+
+ print(
+ "\nNow replace the deployment — destroy the container and start a new one "
+ "from the same image against the same database — then re-run with 'verify'."
+ )
+
+
+def phase_verify(client: httpx.Client, agent: str, thread: str) -> None:
+ """Read the thread back and confirm the deployment can continue it."""
+ print(f"Reading {agent}/{thread} back after the restart\n")
+ code, msgs = _messages(client, agent, thread)
+ check(
+ "thread survived the restart",
+ code == 200 and bool(msgs),
+ f"HTTP {code}, {len(msgs)} message(s) recovered",
+ )
+
+ text = " ".join(str(m.get("content", "")) for m in msgs if isinstance(m, dict))
+ for msg in MESSAGES:
+ check(f"message survived: {msg[:34]!r}", msg in text)
+
+ before = len(msgs)
+ resp = client.post(
+ f"/api/v1/{agent}/chat",
+ json={
+ "content": "Durability probe four: appended after the restart.",
+ "thread_id": thread,
+ },
+ )
+ check(
+ "the new deployment can continue the thread",
+ resp.status_code == 200,
+ f"HTTP {resp.status_code}",
+ )
+
+ code, after = _messages(client, agent, thread)
+ check(
+ "the recovered thread grew",
+ code == 200 and len(after) > before,
+ f"{before} -> {len(after)} message(s)",
+ )
+
+
+def main() -> int:
+ """Parse arguments, run the requested phase, print the verdict."""
+ ap = argparse.ArgumentParser(
+ description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+ )
+ ap.add_argument("phase", choices=("write", "verify"))
+ ap.add_argument("--base-url", default="http://127.0.0.1:8000")
+ ap.add_argument("--api-key", default="")
+ ap.add_argument("--agent", required=True, help="a conversational agent, e.g. ag_chatbot")
+ ap.add_argument("--thread", default="durability-probe")
+ ap.add_argument("--timeout", type=float, default=120.0)
+ args = ap.parse_args()
+
+ client = _client(args.base_url, args.api_key, args.timeout)
+ try:
+ if args.phase == "write":
+ phase_write(client, args.agent, args.thread)
+ else:
+ phase_verify(client, args.agent, args.thread)
+ finally:
+ client.close()
+
+ print()
+ if failures:
+ print(f"DURABILITY FAILED — {len(failures)} check(s)")
+ for failure in failures:
+ print(f" - {failure}")
+ return 1
+ if args.phase == "write":
+ print("Written. Replace the deployment, then run the 'verify' phase.")
+ else:
+ print("DURABILITY PROVEN — conversation state survived losing the container.")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/e2e_verify.py b/scripts/e2e_verify.py
new file mode 100644
index 0000000..d72ba56
--- /dev/null
+++ b/scripts/e2e_verify.py
@@ -0,0 +1,1434 @@
+#!/usr/bin/env python
+"""End-to-end verification harness for a running Agentomatic platform.
+
+Exercises every public surface the platform advertises — platform routes,
+Studio (the exact calls the bundled React UI makes), agents, plugins,
+endpoints, ingestion, pipelines, tasks, control plane, metrics and auth —
+against a live server and reports a pass/fail table.
+
+The harness is deployment-agnostic: point it at ``agentomatic run``, at
+``uvicorn main:app``, or at a container published by ``agentomatic deploy``.
+
+Usage::
+
+ python scripts/e2e_verify.py --base-url http://localhost:8000 \
+ --agent ag_basic --plugin scorer --pipeline basic_flow \
+ --api-key secret --control-token tok --json report.json
+
+Exit code is ``0`` only when every check passes.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import time
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass, field
+from typing import Any
+
+import httpx
+
+#: How many times to wait out a 429 before treating it as a failure.
+_RATE_LIMIT_RETRIES = 3
+#: Cap on a single Retry-After wait, so a long window cannot stall the run.
+_RATE_LIMIT_MAX_WAIT = 65.0
+
+#: Task statuses that mean the work finished successfully.
+_TERMINAL_OK = frozenset({"completed", "succeeded", "success"})
+#: Task statuses that mean the work finished unsuccessfully.
+_TERMINAL_BAD = frozenset({"failed", "error", "cancelled", "canceled"})
+
+#: Every step type the pipeline engine implements. Used to report which
+#: ones a deployment's published pipelines actually exercise.
+_ALL_STEP_TYPES = (
+ "agent",
+ "plugin",
+ "endpoint",
+ "ingestion",
+ "parallel",
+ "map",
+ "transform",
+ "loop",
+ "sub_pipeline",
+)
+
+
+def _collect_step_types(steps: Any, into: set[str]) -> None:
+ """Record every ``step_type`` in a pipeline config, nesting included."""
+ if isinstance(steps, dict):
+ steps = [steps]
+ if not isinstance(steps, list):
+ return
+ for step in steps:
+ if not isinstance(step, dict):
+ continue
+ if step.get("step_type"):
+ into.add(str(step["step_type"]))
+ for nested in ("steps", "body", "step", "branches"):
+ if nested in step:
+ _collect_step_types(step[nested], into)
+
+
+# ---------------------------------------------------------------------------
+# Result plumbing
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class Check:
+ """One verified assertion about the running platform."""
+
+ group: str
+ name: str
+ ok: bool
+ detail: str = ""
+ skipped: bool = False
+
+
+@dataclass
+class Report:
+ """Accumulates checks and renders the final verdict."""
+
+ checks: list[Check] = field(default_factory=list)
+
+ def add(self, group: str, name: str, ok: bool, detail: str = "") -> bool:
+ """Record a check outcome and return ``ok`` for chaining."""
+ self.checks.append(Check(group, name, ok, detail))
+ return ok
+
+ def skip(self, group: str, name: str, why: str) -> None:
+ """Record a check that did not apply to this deployment."""
+ self.checks.append(Check(group, name, True, why, skipped=True))
+
+ @property
+ def failures(self) -> list[Check]:
+ """Return every failed check."""
+ return [c for c in self.checks if not c.ok]
+
+ def render(self) -> str:
+ """Render a human-readable summary table."""
+ lines: list[str] = []
+ groups: dict[str, list[Check]] = {}
+ for c in self.checks:
+ groups.setdefault(c.group, []).append(c)
+ for group, items in groups.items():
+ passed = sum(1 for i in items if i.ok and not i.skipped)
+ skipped = sum(1 for i in items if i.skipped)
+ failed = sum(1 for i in items if not i.ok)
+ status = "FAIL" if failed else "PASS"
+ lines.append(
+ f"[{status}] {group:<22} {passed:>3} passed"
+ + (f", {skipped} skipped" if skipped else "")
+ + (f", {failed} FAILED" if failed else "")
+ )
+ for i in items:
+ if not i.ok:
+ lines.append(f" ✗ {i.name}: {i.detail}")
+ total = len(self.checks)
+ failed = len(self.failures)
+ lines.append("")
+ lines.append(f"TOTAL: {total} checks, {total - failed} passed, {failed} failed")
+ return "\n".join(lines)
+
+
+# ---------------------------------------------------------------------------
+# Verifier
+# ---------------------------------------------------------------------------
+
+
+class Verifier:
+ """Drives every check against one base URL."""
+
+ def __init__(
+ self,
+ base_url: str,
+ api_key: str = "",
+ control_token: str = "",
+ agent: str = "ag_basic",
+ plugin: str = "scorer",
+ pipeline: str = "basic_flow",
+ endpoint: str = "echo",
+ ingestor: str = "",
+ timeout: float = 30.0,
+ expect_auth: bool = False,
+ expect_studio: bool = True,
+ ) -> None:
+ self.base = base_url.rstrip("/")
+ self.api_key = api_key
+ self.control_token = control_token
+ self.agent = agent
+ self.plugin = plugin
+ self.pipeline = pipeline
+ self.endpoint = endpoint
+ self.ingestor = ingestor
+ self.expect_auth = expect_auth
+ self.expect_studio = expect_studio
+ self.report = Report()
+ #: Set by ``verify_agent_rest``'s thread probe. A deployment with
+ #: no store is a legitimate posture, so store-dependent checks
+ #: report as skipped rather than failing.
+ self.thread_store_available = True
+ headers: dict[str, str] = {"Content-Type": "application/json"}
+ if api_key:
+ # Mirror exactly what the Studio bundle sends.
+ headers["X-Api-Key"] = api_key
+ headers["Authorization"] = f"Bearer {api_key}"
+ self.client = httpx.Client(base_url=self.base, headers=headers, timeout=timeout)
+
+ # -- helpers ---------------------------------------------------------
+
+ def _req(
+ self,
+ method: str,
+ path: str,
+ *,
+ json_body: Any = None,
+ headers: dict[str, str] | None = None,
+ honour_retry_after: bool = True,
+ ) -> httpx.Response | None:
+ """Issue a request, returning ``None`` on transport failure.
+
+ When the deployment enables rate limiting, this harness is itself a
+ burst of traffic from one IP. A ``429`` is the limiter behaving
+ correctly, so wait out ``Retry-After`` and try again rather than
+ reporting a false failure. Set ``honour_retry_after=False`` when the
+ ``429`` is the thing under test.
+ """
+ for attempt in range(_RATE_LIMIT_RETRIES):
+ try:
+ resp = self.client.request(method, path, json=json_body, headers=headers)
+ except Exception: # noqa: BLE001 - reported as a failed check
+ return None
+ if resp.status_code != 429 or not honour_retry_after:
+ return resp
+ if attempt == _RATE_LIMIT_RETRIES - 1:
+ return resp
+ delay = float(resp.headers.get("Retry-After") or 1)
+ time.sleep(min(delay, _RATE_LIMIT_MAX_WAIT) + 0.5)
+ return None
+
+ def check(
+ self,
+ group: str,
+ name: str,
+ method: str,
+ path: str,
+ *,
+ json_body: Any = None,
+ expect: tuple[int, ...] = (200,),
+ headers: dict[str, str] | None = None,
+ validate: Any = None,
+ ) -> Any:
+ """Call one route, assert its status, and optionally validate the body.
+
+ Args:
+ group: Report grouping label.
+ name: Human-readable check name.
+ method: HTTP method.
+ path: Path relative to the base URL.
+ json_body: Optional JSON request body.
+ expect: Acceptable status codes.
+ headers: Extra request headers.
+ validate: Optional ``callable(payload) -> str``; a non-empty
+ return value marks the check failed with that message.
+
+ Returns:
+ The decoded JSON payload when available, else ``None``.
+ """
+ resp = self._req(method, path, json_body=json_body, headers=headers)
+ if resp is None:
+ self.report.add(group, name, False, f"{method} {path} — transport error")
+ return None
+ if resp.status_code not in expect:
+ body = resp.text[:300].replace("\n", " ")
+ self.report.add(
+ group,
+ name,
+ False,
+ f"{method} {path} → {resp.status_code} (want {expect}): {body}",
+ )
+ return None
+ payload: Any = None
+ if resp.content:
+ try:
+ payload = resp.json()
+ except Exception: # noqa: BLE001 - non-JSON bodies are fine
+ payload = None
+ if validate is not None and payload is not None:
+ problem = validate(payload)
+ if problem:
+ self.report.add(group, name, False, f"{method} {path} — {problem}")
+ return payload
+ self.report.add(group, name, True)
+ return payload
+
+ def sse(
+ self,
+ group: str,
+ name: str,
+ path: str,
+ body: dict[str, Any],
+ *,
+ want_events: tuple[str, ...] = (),
+ ) -> list[dict[str, Any]]:
+ """Stream a POST SSE endpoint exactly as the Studio bundle does."""
+ events: list[dict[str, Any]] = []
+ raw_lines: list[str] = []
+ for attempt in range(_RATE_LIMIT_RETRIES):
+ events, raw_lines, retry_after = self._stream_once(group, name, path, body)
+ if retry_after is None:
+ break
+ if attempt == _RATE_LIMIT_RETRIES - 1:
+ self.report.add(group, name, False, f"{path} — still rate limited")
+ return events
+ time.sleep(min(retry_after, _RATE_LIMIT_MAX_WAIT) + 0.5)
+ else: # pragma: no cover - loop always breaks or returns
+ return events
+ if not events:
+ self.report.add(group, name, False, f"{path} — no SSE data frames ({raw_lines[:4]})")
+ return events
+ seen = {str(e.get("event") or e.get("type") or "") for e in events}
+ missing = [w for w in want_events if w not in seen]
+ if missing:
+ self.report.add(
+ group, name, False, f"{path} — missing events {missing}; got {sorted(seen)}"
+ )
+ return events
+ self.report.add(group, name, True)
+ return events
+
+ def _stream_once(
+ self, group: str, name: str, path: str, body: dict[str, Any]
+ ) -> tuple[list[dict[str, Any]], list[str], float | None]:
+ """Read one SSE attempt.
+
+ Returns:
+ ``(events, raw_lines, retry_after)``. ``retry_after`` is set only
+ when the deployment rate-limited this attempt, in which case the
+ caller should wait and retry rather than report a failure.
+ """
+ events: list[dict[str, Any]] = []
+ raw_lines: list[str] = []
+ try:
+ with self.client.stream(
+ "POST",
+ path,
+ json=body,
+ headers={"Accept": "text/event-stream"},
+ ) as resp:
+ if resp.status_code == 429:
+ return events, raw_lines, float(resp.headers.get("Retry-After") or 1)
+ if resp.status_code != 200:
+ text = resp.read().decode("utf-8", "replace")[:300]
+ self.report.add(group, name, False, f"{path} → {resp.status_code}: {text}")
+ return events, raw_lines, None
+ ctype = resp.headers.get("content-type", "")
+ if "text/event-stream" not in ctype:
+ self.report.add(group, name, False, f"{path} — content-type {ctype!r}")
+ return events, raw_lines, None
+ for line in resp.iter_lines():
+ raw_lines.append(line)
+ if not line.startswith("data:"):
+ continue
+ data = line[5:].strip()
+ if data == "[DONE]":
+ break
+ try:
+ events.append(json.loads(data))
+ except Exception: # noqa: BLE001 - captured below
+ pass
+ except Exception as exc: # noqa: BLE001
+ self.report.add(group, name, False, f"{path} — {type(exc).__name__}: {exc}")
+ return events, raw_lines, None
+
+ def await_task(self, group: str, name: str, submitted: Any, *, timeout: float = 30.0) -> Any:
+ """Poll a 202-submitted task to a terminal state and return its record.
+
+ Args:
+ group: Report grouping label.
+ name: Human-readable check name.
+ submitted: Decoded body of the 202 response.
+ timeout: Seconds to wait for a terminal status.
+
+ Returns:
+ The final task record, or ``None`` when it never completed.
+ """
+ task_id = None
+ if isinstance(submitted, dict):
+ task_id = submitted.get("id") or submitted.get("task_id")
+ if not task_id:
+ self.report.add(group, name, False, f"no task id in {submitted}")
+ return None
+ deadline = time.time() + timeout
+ record: Any = None
+ while time.time() < deadline:
+ resp = self._req("GET", f"/api/v1/tasks/{task_id}")
+ if resp is not None and resp.status_code == 200:
+ try:
+ record = resp.json()
+ except Exception: # noqa: BLE001
+ record = None
+ status = (record or {}).get("status")
+ if status in _TERMINAL_OK or status in _TERMINAL_BAD:
+ break
+ time.sleep(0.4)
+ status = (record or {}).get("status")
+ if status not in _TERMINAL_OK:
+ err = (record or {}).get("error")
+ self.report.add(group, name, False, f"status={status!r} error={err!r}")
+ return record
+ self.report.add(group, name, True)
+ return record
+
+ # -- groups ----------------------------------------------------------
+
+ def verify_platform(self) -> None:
+ """Health, readiness, status, OpenAPI and docs."""
+ g = "platform"
+ self.check(
+ g,
+ "health",
+ "GET",
+ "/health",
+ validate=lambda p: "" if p.get("status") else "no status field",
+ )
+ self.check(g, "ready", "GET", "/ready")
+ self.check(g, "readiness", "GET", "/readiness")
+ self.check(
+ g,
+ "status",
+ "GET",
+ "/status",
+ validate=lambda p: "" if isinstance(p, dict) else "not an object",
+ )
+ self.check(
+ g,
+ "api status",
+ "GET",
+ "/api/v1/status",
+ validate=lambda p: "" if isinstance(p, dict) else "not an object",
+ )
+
+ def _openapi(p: Any) -> str:
+ if not isinstance(p, dict) or "paths" not in p:
+ return "no paths"
+ if not p["paths"]:
+ return "empty paths"
+ return ""
+
+ self.check(g, "openapi.json", "GET", "/openapi.json", validate=_openapi)
+ for doc in ("/docs", "/redoc"):
+ resp = self._req("GET", doc)
+ ok = resp is not None and resp.status_code == 200
+ self.report.add(
+ g,
+ f"docs {doc}",
+ ok,
+ "" if ok else f"{doc} → {getattr(resp, 'status_code', 'transport error')}",
+ )
+
+ def _agents(p: Any) -> str:
+ items = p.get("agents") if isinstance(p, dict) else p
+ if not items:
+ return "no agents listed"
+ return ""
+
+ self.check(g, "agents registry", "GET", "/api/v1/agents", validate=_agents)
+
+ def verify_studio(self) -> None:
+ """Every call the bundled Studio React client makes."""
+ g = "studio"
+ if not self.expect_studio:
+ self.report.skip(g, "studio disabled", "AGENTOMATIC_ENABLE_STUDIO=0")
+ return
+ a = self.agent
+
+ def _info(p: Any) -> str:
+ # Field names the Studio bundle reads verbatim (ConnectionSetup,
+ # ControlPlaneView): info.version, info.platform_title, info.agent_count.
+ missing = [k for k in ("version", "platform_title", "agent_count") if k not in p]
+ return f"missing {missing}" if missing else ""
+
+ self.check(g, "GET /studio/info", "GET", "/studio/info", validate=_info)
+
+ def _agents(p: Any) -> str:
+ items = p if isinstance(p, list) else p.get("agents", [])
+ if not items:
+ return "no agents"
+ first = items[0] if isinstance(items, list) else None
+ if isinstance(first, dict):
+ # The bundle reads these fields directly off each entry.
+ missing = [k for k in ("name", "slug", "framework") if k not in first]
+ if missing:
+ return f"agent entry missing {missing}"
+ return ""
+
+ self.check(g, "GET /studio/agents", "GET", "/studio/agents", validate=_agents)
+
+ def _graph(p: Any) -> str:
+ if not isinstance(p, dict):
+ return "not an object"
+ if "nodes" not in p or "edges" not in p:
+ return f"missing nodes/edges, got {sorted(p)[:8]}"
+ return ""
+
+ self.check(g, "GET graph", "GET", f"/studio/agents/{a}/graph", validate=_graph)
+ self.check(g, "GET schemas", "GET", f"/studio/agents/{a}/schemas")
+ self.check(g, "GET config", "GET", f"/studio/agents/{a}/config")
+
+ def _run(p: Any) -> str:
+ # useStudioStore matches runs on `run.id`; RunInfo also carries
+ # agent_name/status/created_at, which the runs list renders.
+ if not isinstance(p, dict):
+ return "not an object"
+ missing = [k for k in ("id", "agent_name", "status", "created_at") if k not in p]
+ return f"missing {missing}" if missing else ""
+
+ run = self.check(
+ g,
+ "POST runs",
+ "POST",
+ f"/studio/agents/{a}/runs",
+ json_body={"query": "e2e studio run", "user_id": "e2e"},
+ validate=_run,
+ )
+ self.check(g, "GET runs list", "GET", f"/studio/agents/{a}/runs?limit=50")
+ if isinstance(run, dict) and run.get("id"):
+ self.check(g, "GET run by id", "GET", f"/studio/agents/{a}/runs/{run['id']}")
+
+ thread_id = run.get("thread_id") if isinstance(run, dict) else None
+ if thread_id:
+ self.check(
+ g, "GET thread state", "GET", f"/studio/agents/{a}/threads/{thread_id}/state"
+ )
+ self.check(
+ g,
+ "POST thread state",
+ "POST",
+ f"/studio/agents/{a}/threads/{thread_id}/state",
+ json_body={"updates": {"e2e": True}},
+ )
+ self.check(
+ g, "GET thread history", "GET", f"/studio/agents/{a}/threads/{thread_id}/history"
+ )
+ else:
+ self.report.skip(g, "thread state/history", "run returned no thread_id")
+
+ self.sse(
+ g,
+ "POST runs/stream (SSE)",
+ f"/studio/agents/{a}/runs/stream",
+ {"query": "e2e stream", "user_id": "e2e"},
+ want_events=("run_start", "run_complete"),
+ )
+
+ resp = self._req("GET", "/studio/ui")
+ ok = resp is not None and resp.status_code in (200, 307, 308)
+ self.report.add(
+ g,
+ "GET /studio/ui",
+ ok,
+ "" if ok else f"→ {getattr(resp, 'status_code', 'transport error')}",
+ )
+ # The SPA bundle must actually be served, not just the route exist.
+ resp = self._req("GET", "/studio/ui/index.html")
+ ok = resp is not None and resp.status_code == 200 and b"
None:
+ """The documented agent REST contract."""
+ g = "agent-rest"
+ a = self.agent
+ base = f"/api/v1/{a}"
+
+ def _invoke(p: Any) -> str:
+ if not isinstance(p, dict):
+ return "not an object"
+ if "result" not in p and "output" not in p and "response" not in p:
+ return f"no result/output/response key: {sorted(p)[:8]}"
+ return ""
+
+ self.check(
+ g,
+ "POST invoke",
+ "POST",
+ f"{base}/invoke",
+ json_body={"query": "hello e2e"},
+ validate=_invoke,
+ )
+ # The wire field is `query`; posting `current_query` must be rejected.
+ self.check(
+ g,
+ "invoke rejects current_query",
+ "POST",
+ f"{base}/invoke",
+ json_body={"current_query": "hello"},
+ expect=(422,),
+ )
+ self.check(
+ g,
+ "POST chat",
+ "POST",
+ f"{base}/chat",
+ json_body={"content": "hi there"},
+ )
+ submitted = self.check(
+ g,
+ "POST invoke/batch (202)",
+ "POST",
+ f"{base}/invoke/batch",
+ json_body={"inputs": [{"query": "one"}, {"query": "two"}]},
+ expect=(202,),
+ )
+ record = self.await_task(g, "invoke/batch completes", submitted)
+ results = (record or {}).get("result")
+ ok = isinstance(results, list) and len(results) == 2
+ self.report.add(
+ g,
+ "invoke/batch returns both items",
+ ok,
+ "" if ok else f"batch result was {results!r}",
+ )
+ # A batch body that names the item list wrongly, or carries no items,
+ # must be rejected — never accepted as a zero-item "succeeded" batch.
+ self.check(
+ g,
+ "batch rejects unknown item key",
+ "POST",
+ f"{base}/invoke/batch",
+ json_body={"items": [{"query": "one"}]},
+ expect=(422,),
+ )
+ self.check(
+ g,
+ "batch rejects empty inputs",
+ "POST",
+ f"{base}/invoke/batch",
+ json_body={"inputs": []},
+ expect=(422,),
+ )
+ self.check(g, "GET health", "GET", f"{base}/health")
+ self.check(g, "GET card", "GET", f"{base}/card")
+ self.check(g, "GET config", "GET", f"{base}/config")
+ self.check(g, "GET prompts", "GET", f"{base}/prompts")
+
+ self.sse(
+ g,
+ "POST invoke/stream (SSE)",
+ f"{base}/invoke/stream",
+ {"query": "stream me"},
+ )
+
+ # Threads: the Studio client drives this whole lifecycle. A deployment
+ # with no store configured is a legitimate posture (the platform says
+ # so with a 400), not a failure — report the whole group as skipped
+ # rather than as broken, and say why.
+ probe = self._req(
+ "POST",
+ f"{base}/threads",
+ json_body={"user_id": "e2e-user", "title": "e2e thread"},
+ )
+ if probe is not None and probe.status_code == 400 and "storage" in probe.text.lower():
+ self.thread_store_available = False
+ self.report.skip(
+ g,
+ "thread lifecycle",
+ "no store configured (set DATABASE_URL / AGENTOMATIC_LOGS_HISTORY)",
+ )
+ self.report.skip(g, "optimization-runs", "no store configured")
+ return
+ thread = self.check(
+ g,
+ "POST threads",
+ "POST",
+ f"{base}/threads",
+ json_body={"user_id": "e2e-user", "title": "e2e thread"},
+ )
+ tid = None
+ if isinstance(thread, dict):
+ payload = thread.get("thread") or thread
+ tid = payload.get("id") or payload.get("thread_id")
+ if not tid:
+ self.report.add(g, "thread id present", False, f"no id in {thread}")
+ return
+ self.report.add(g, "thread id present", True)
+ self.check(g, "GET optimization-runs", "GET", f"{base}/optimization-runs")
+ self.check(g, "GET threads", "GET", f"{base}/threads")
+ self.check(g, "GET thread", "GET", f"{base}/threads/{tid}")
+ self.check(
+ g,
+ "PATCH thread",
+ "PATCH",
+ f"{base}/threads/{tid}",
+ json_body={"title": "renamed"},
+ )
+ # Post a message through chat so the thread has history to read back.
+ self.check(
+ g,
+ "chat into thread",
+ "POST",
+ f"{base}/chat",
+ json_body={"content": "remember this", "thread_id": tid, "user_id": "e2e-user"},
+ )
+ self.check(g, "GET messages", "GET", f"{base}/threads/{tid}/messages")
+ self.check(g, "GET summary", "GET", f"{base}/threads/{tid}/summary")
+ self.check(g, "GET lineage", "GET", f"{base}/threads/{tid}/lineage")
+ self.check(g, "GET pending approvals", "GET", f"{base}/threads/{tid}/pending")
+ self.check(
+ g,
+ "POST fork",
+ "POST",
+ f"{base}/threads/{tid}/fork",
+ json_body={"message_index": 0},
+ )
+ self.check(
+ g,
+ "POST feedback",
+ "POST",
+ f"{base}/feedback",
+ json_body={"thread_id": tid, "rating": 5, "comment": "e2e"},
+ expect=(200, 201),
+ )
+ self.check(g, "GET feedback", "GET", f"{base}/feedback")
+ self.check(g, "GET feedback/export", "GET", f"{base}/feedback/export")
+ self.check(g, "DELETE messages", "DELETE", f"{base}/threads/{tid}/messages")
+ self.check(g, "DELETE thread", "DELETE", f"{base}/threads/{tid}")
+
+ def verify_a2a(self) -> None:
+ """Agent-to-Agent discovery card and task protocol."""
+ g = "a2a"
+ # The well-known card is how a peer agent discovers this platform.
+ self.check(
+ g,
+ "GET /.well-known/agent.json",
+ "GET",
+ "/.well-known/agent.json",
+ validate=lambda p: "" if isinstance(p, dict) and p else "empty card",
+ )
+ base = f"/api/v1/{self.agent}/a2a"
+ task = self.check(
+ g,
+ "POST a2a/tasks",
+ "POST",
+ f"{base}/tasks",
+ json_body={
+ "message": {"role": "user", "parts": [{"type": "text", "text": "a2a hello"}]}
+ },
+ expect=(200, 201, 202),
+ )
+ tid = None
+ if isinstance(task, dict):
+ tid = task.get("id") or task.get("task_id") or (task.get("task") or {}).get("id")
+ if tid:
+ self.report.add(g, "a2a task id", True)
+ self.check(g, "GET a2a task", "GET", f"{base}/tasks/{tid}")
+ self.check(
+ g,
+ "POST a2a cancel",
+ "POST",
+ f"{base}/tasks/{tid}/cancel",
+ expect=(200, 202, 409),
+ )
+ else:
+ self.report.add(g, "a2a task id", False, f"no task id in {task}")
+
+ def verify_plugins(self) -> None:
+ """Plugin registry, model card and inference routes."""
+ g = "plugins"
+ p = self.plugin
+
+ def _list(payload: Any) -> str:
+ items = payload if isinstance(payload, list) else payload.get("plugins", [])
+ if not items:
+ return "no plugins listed"
+ return ""
+
+ self.check(g, "GET /api/v1/plugins", "GET", "/api/v1/plugins", validate=_list)
+ if not p:
+ self.report.skip(g, "plugin routes", "no plugin configured")
+ return
+ self.check(g, "GET model_card", "GET", f"/api/v1/plugins/{p}/model_card")
+ self.check(g, "GET plugin health", "GET", f"/api/v1/plugins/{p}/health")
+ self.check(
+ g,
+ "POST predict",
+ "POST",
+ f"/api/v1/plugins/{p}/predict",
+ json_body={"text": "a reasonably long sentence"},
+ )
+ submitted = self.check(
+ g,
+ "POST predict/batch (202)",
+ "POST",
+ f"/api/v1/plugins/{p}/predict/batch",
+ json_body={"inputs": [{"text": "one"}, {"text": "two"}]},
+ expect=(202,),
+ )
+ self.await_task(g, "predict/batch completes", submitted)
+ self.check(
+ g,
+ "POST plugin reload",
+ "POST",
+ f"/api/v1/plugins/{p}/reload",
+ expect=(200, 202),
+ )
+
+ def verify_endpoints(self) -> None:
+ """Custom endpoint mounting and invocation."""
+ g = "endpoints"
+ self.check(g, "GET /api/v1/endpoints", "GET", "/api/v1/endpoints")
+ e = self.endpoint
+ if not e:
+ self.report.skip(g, "endpoint routes", "no endpoint configured")
+ return
+ self.check(g, "GET endpoint info", "GET", f"/api/v1/endpoints/{e}/info")
+ self.check(g, "GET endpoint health", "GET", f"/api/v1/endpoints/{e}/health")
+ self.check(
+ g,
+ "POST endpoint call",
+ "POST",
+ f"/api/v1/endpoints/{e}/call",
+ json_body={"payload": {"text": "shout"}},
+ )
+
+ def verify_ingestion(self) -> None:
+ """Ingestion registry and run routes."""
+ g = "ingestion"
+ self.check(g, "GET /api/v1/ingestion", "GET", "/api/v1/ingestion")
+ # The Studio bundle uses the /ingestors alias — both must exist.
+ self.check(g, "GET /api/v1/ingestors", "GET", "/api/v1/ingestors")
+ i = self.ingestor
+ if not i:
+ self.report.skip(g, "ingestor routes", "no ingestor configured")
+ return
+ self.check(g, "GET ingestor info", "GET", f"/api/v1/ingestion/{i}/info")
+ self.check(g, "GET ingestor health", "GET", f"/api/v1/ingestion/{i}/health")
+
+ def _run(p: Any) -> str:
+ if not isinstance(p, dict):
+ return "not an object"
+ if p.get("status") not in ("succeeded", "success", "completed", "partial"):
+ return f"status={p.get('status')!r} errors={p.get('errors')!r}"
+ return ""
+
+ self.check(
+ g,
+ "POST ingestor run",
+ "POST",
+ f"/api/v1/ingestion/{i}/run",
+ json_body={"source": "inline://e2e verification document"},
+ validate=_run,
+ )
+
+ def verify_pipelines(self) -> None:
+ """Pipeline discovery, validation, visualisation and execution."""
+ g = "pipelines"
+
+ def _list(payload: Any) -> str:
+ items = payload if isinstance(payload, list) else payload.get("pipelines", [])
+ if not items:
+ return "no pipelines listed"
+ return ""
+
+ self.check(g, "GET /api/v1/pipelines", "GET", "/api/v1/pipelines", validate=_list)
+ p = self.pipeline
+ if not p:
+ self.report.skip(g, "pipeline routes", "no pipeline configured")
+ return
+ self.check(g, "GET pipeline config", "GET", f"/api/v1/pipelines/{p}/config")
+ self.check(g, "GET pipeline validate", "GET", f"/api/v1/pipelines/{p}/validate")
+
+ def _viz(payload: Any) -> str:
+ if not isinstance(payload, dict) or not payload.get("mermaid"):
+ return "no mermaid field"
+ return ""
+
+ self.check(
+ g, "GET pipeline visualize", "GET", f"/api/v1/pipelines/{p}/visualize", validate=_viz
+ )
+ self.check(
+ g,
+ "POST pipeline run",
+ "POST",
+ f"/api/v1/pipelines/{p}/run",
+ json_body={"input": {"query": "pipeline e2e"}},
+ )
+ self.check(
+ g,
+ "POST validate-draft",
+ "POST",
+ "/api/v1/pipelines/validate-draft",
+ json_body={"yaml": "name: draft_check\nsteps:\n - agent: " + self.agent + "\n"},
+ )
+
+ def verify_every_pipeline(self) -> None:
+ """Run *every* published pipeline, not just the sampled one.
+
+ ``verify_pipelines`` proves the routes work against one pipeline. A
+ deployment usually publishes several, each built from different step
+ types, and a step type that only ever validates is not a step type
+ that runs. This executes them all and reports which step types were
+ actually exercised.
+ """
+ g = "pipelines-all"
+ listing = self.check(g, "GET /api/v1/pipelines", "GET", "/api/v1/pipelines")
+ if listing is None:
+ return
+ items = listing if isinstance(listing, list) else listing.get("pipelines", [])
+ names = [i["name"] if isinstance(i, dict) else str(i) for i in items]
+ if not names:
+ self.report.skip(g, "run every pipeline", "no pipelines published")
+ return
+
+ seen: set[str] = set()
+ for name in sorted(names):
+ cfg = self.check(g, f"config: {name}", "GET", f"/api/v1/pipelines/{name}/config")
+ if isinstance(cfg, dict):
+ _collect_step_types(cfg.get("steps") or [], seen)
+
+ def _ran(payload: Any) -> str:
+ if not isinstance(payload, dict):
+ return "non-dict result"
+ if payload.get("status") != "success":
+ return f"status={payload.get('status')} {str(payload.get('error'))[:120]}"
+ bad = {
+ n: st.get("status")
+ for n, st in (payload.get("steps") or {}).items()
+ if st.get("status") not in ("success", "skipped")
+ }
+ return f"steps not ok: {bad}" if bad else ""
+
+ self.check(
+ g,
+ f"run: {name}",
+ "POST",
+ f"/api/v1/pipelines/{name}/run",
+ json_body={"input": {"query": "one two three four five"}},
+ validate=_ran,
+ )
+
+ for step_type in sorted(seen):
+ self.report.add(g, f"step type executed: {step_type}", True)
+ missing = sorted(set(_ALL_STEP_TYPES) - seen)
+ if missing:
+ self.report.skip(
+ g, "full step-type coverage", f"not published by this deployment: {missing}"
+ )
+
+ def verify_isolation(self) -> None:
+ """Concurrent callers must never see each other's data.
+
+ Agents are singletons: every request gets the same instance. A class
+ agent that parks per-run data on ``self`` instead of in its state
+ dataclass serves caller A's answer to caller B — a failure that is
+ invisible to sequential testing and is the worst kind to ship.
+
+ Each request carries a marker unique to it; a response containing
+ somebody else's marker is a leak.
+ """
+ g = "isolation"
+ fanout = 24
+ run = f"{int(time.time()):x}"
+
+ def marker(i: int) -> str:
+ return f"MK{i:04d}{run}ZQ"
+
+ def one(i: int) -> tuple[int, int, str]:
+ resp = self._req(
+ "POST",
+ f"/api/v1/{self.agent}/invoke",
+ json_body={"query": f"Reply with exactly {marker(i)} and nothing else."},
+ )
+ if resp is None:
+ return i, 0, ""
+ return i, resp.status_code, resp.text
+
+ with ThreadPoolExecutor(max_workers=fanout) as pool:
+ results = list(pool.map(one, range(fanout)))
+
+ bad = [(i, code) for i, code, _ in results if code != 200]
+ self.report.add(
+ g, f"{fanout} concurrent invokes all succeeded", not bad, f"non-200: {bad[:5]}"
+ )
+
+ leaked = [
+ (marker(i), [marker(j) for j in range(fanout) if j != i and marker(j) in body][:3])
+ for i, code, body in results
+ if code == 200 and any(marker(j) in body for j in range(fanout) if j != i)
+ ]
+ self.report.add(
+ g, "no response carried another caller's marker", not leaked, str(leaked[:3])
+ )
+
+ if not self.thread_store_available:
+ self.report.skip(g, "concurrent threads stay separate", "no thread store")
+ return
+
+ def one_chat(i: int) -> tuple[int, int, Any]:
+ resp = self._req(
+ "POST",
+ f"/api/v1/{self.agent}/chat",
+ json_body={"content": f"Token {marker(i)}", "thread_id": f"iso-{marker(i)}"},
+ )
+ if resp is None:
+ return i, 0, None
+ try:
+ return i, resp.status_code, resp.json()
+ except Exception: # noqa: BLE001 - non-JSON body is a failure below
+ return i, resp.status_code, None
+
+ with ThreadPoolExecutor(max_workers=fanout) as pool:
+ chats = list(pool.map(one_chat, range(fanout)))
+
+ wrong = [
+ (marker(i), (body or {}).get("thread_id"))
+ for i, code, body in chats
+ if code == 200 and (body or {}).get("thread_id") != f"iso-{marker(i)}"
+ ]
+ self.report.add(g, "each reply came back on its own thread", not wrong, str(wrong[:3]))
+
+ borrowed = [
+ (marker(i), (body or {}).get("history_loaded"))
+ for i, code, body in chats
+ if code == 200 and ((body or {}).get("history_loaded") or 0) != 0
+ ]
+ self.report.add(
+ g, "no fresh thread picked up another's history", not borrowed, str(borrowed[:3])
+ )
+
+ def verify_tasks(self) -> None:
+ """Async task manager routes."""
+ g = "tasks"
+ self.check(g, "GET /api/v1/tasks", "GET", "/api/v1/tasks")
+ created = self.check(
+ g,
+ "POST agent invoke/async",
+ "POST",
+ f"/api/v1/{self.agent}/invoke/async",
+ json_body={"query": "async e2e"},
+ expect=(200, 201, 202),
+ )
+ task_id = None
+ if isinstance(created, dict):
+ task_id = created.get("task_id") or created.get("id")
+ if not task_id:
+ self.report.add(g, "async task id", False, f"no task id in {created}")
+ return
+ self.report.add(g, "async task id", True)
+ self.check(g, "GET task", "GET", f"/api/v1/tasks/{task_id}")
+ self.await_task(g, "async task completes", created)
+ self.check(g, "GET task result", "GET", f"/api/v1/tasks/{task_id}/result", expect=(200,))
+ self.check(g, "DELETE task", "DELETE", f"/api/v1/tasks/{task_id}", expect=(200, 204))
+
+ def verify_control_plane(self) -> None:
+ """Control-plane read and mutate routes."""
+ g = "control-plane"
+ info = self._req("GET", "/api/v1/control")
+ if info is None or info.status_code == 404:
+ self.report.skip(g, "control plane", "not enabled on this deployment")
+ return
+ self.check(g, "GET /api/v1/control", "GET", "/api/v1/control")
+ self.check(g, "GET control/agents", "GET", "/api/v1/control/agents")
+ self.check(g, "GET control/agent", "GET", f"/api/v1/control/agents/{self.agent}")
+ self.check(g, "GET control/endpoints", "GET", "/api/v1/control/endpoints")
+ self.check(g, "GET control/connections", "GET", "/api/v1/control/connections")
+ self.check(g, "GET control/health", "GET", "/api/v1/control/health")
+ self.check(g, "GET control/config", "GET", "/api/v1/control/config")
+ self.check(g, "GET control/metrics/summary", "GET", "/api/v1/control/metrics/summary")
+
+ ctl = {"X-Control-Token": self.control_token} if self.control_token else None
+ self.check(
+ g,
+ "POST agent disable",
+ "POST",
+ f"/api/v1/control/agents/{self.agent}/disable",
+ headers=ctl,
+ )
+ # A disabled agent must actually stop serving traffic.
+ resp = self._req("POST", f"/api/v1/{self.agent}/invoke", json_body={"query": "x"})
+ ok = resp is not None and resp.status_code in (403, 404, 503)
+ self.report.add(
+ g,
+ "disabled agent refuses traffic",
+ ok,
+ "" if ok else f"invoke → {getattr(resp, 'status_code', 'transport error')}",
+ )
+ self.check(
+ g,
+ "POST agent enable",
+ "POST",
+ f"/api/v1/control/agents/{self.agent}/enable",
+ headers=ctl,
+ )
+ resp = self._req("POST", f"/api/v1/{self.agent}/invoke", json_body={"query": "x"})
+ ok = resp is not None and resp.status_code == 200
+ self.report.add(
+ g,
+ "re-enabled agent serves traffic",
+ ok,
+ "" if ok else f"invoke → {getattr(resp, 'status_code', 'transport error')}",
+ )
+ self.check(
+ g,
+ "POST maintenance on",
+ "POST",
+ "/api/v1/control/maintenance",
+ json_body={"enabled": True},
+ headers=ctl,
+ )
+ self.check(
+ g,
+ "POST maintenance off",
+ "POST",
+ "/api/v1/control/maintenance",
+ json_body={"enabled": False},
+ headers=ctl,
+ )
+
+ def verify_metrics(self) -> None:
+ """Prometheus exposition."""
+ g = "metrics"
+ resp = self._req("GET", "/metrics")
+ if resp is None:
+ self.report.add(g, "GET /metrics", False, "transport error")
+ return
+ if resp.status_code == 404:
+ self.report.skip(g, "GET /metrics", "metrics disabled on this deployment")
+ return
+ ok = resp.status_code == 200 and b"# HELP" in resp.content
+ self.report.add(
+ g,
+ "GET /metrics",
+ ok,
+ "" if ok else f"→ {resp.status_code}, body starts {resp.content[:80]!r}",
+ )
+ ok = b"agentomatic" in resp.content.lower()
+ self.report.add(
+ g, "metrics include agentomatic series", ok, "" if ok else "no agentomatic_* series"
+ )
+
+ def verify_auth(self) -> None:
+ """Auth enforcement, when the deployment is configured for it."""
+ g = "auth"
+ if not self.expect_auth:
+ self.report.skip(g, "auth enforcement", "deployment runs without auth")
+ return
+ anon = httpx.Client(base_url=self.base, timeout=15.0)
+
+ def _anon(headers: dict[str, str] | None = None) -> httpx.Response:
+ """POST as an anonymous caller, waiting out the shared rate limit.
+
+ The anonymous client shares this harness's source IP, so it shares
+ the limiter's per-IP budget — a 429 here says nothing about auth.
+ """
+ for attempt in range(_RATE_LIMIT_RETRIES):
+ got = anon.post(
+ f"/api/v1/{self.agent}/invoke", json={"query": "x"}, headers=headers
+ )
+ if got.status_code != 429 or attempt == _RATE_LIMIT_RETRIES - 1:
+ return got
+ time.sleep(
+ min(float(got.headers.get("Retry-After") or 1), _RATE_LIMIT_MAX_WAIT) + 0.5
+ )
+ return got
+
+ try:
+ # Protected route must reject an anonymous caller.
+ resp = _anon()
+ ok = resp.status_code in (401, 403)
+ self.report.add(
+ g, "anonymous invoke rejected", ok, "" if ok else f"→ {resp.status_code}"
+ )
+ # A wrong key must also be rejected.
+ resp = _anon({"X-Api-Key": "definitely-wrong"})
+ ok = resp.status_code in (401, 403)
+ self.report.add(g, "bad key rejected", ok, "" if ok else f"→ {resp.status_code}")
+ # Every probe route must stay open: an orchestrator carries no
+ # credentials, and a readiness probe that 401s keeps a pod out of
+ # service for good while the platform looks healthy in its logs.
+ for path in ("/health", "/ready", "/readiness"):
+ resp = anon.get(path)
+ ok = resp.status_code == 200
+ self.report.add(
+ g, f"{path} stays public", ok, "" if ok else f"→ {resp.status_code}"
+ )
+ except Exception as exc: # noqa: BLE001
+ self.report.add(g, "auth checks", False, f"{type(exc).__name__}: {exc}")
+ finally:
+ anon.close()
+ # And the correct key must work.
+ self.check(
+ g,
+ "valid key accepted",
+ "POST",
+ f"/api/v1/{self.agent}/invoke",
+ json_body={"query": "authed"},
+ )
+
+ def verify_error_contract(self) -> None:
+ """Unknown resources must 404 cleanly, never 500."""
+ g = "errors"
+ for name, method, path, body in (
+ ("unknown agent", "POST", "/api/v1/definitely_not_an_agent/invoke", {"query": "x"}),
+ ("unknown plugin", "GET", "/api/v1/plugins/nope/model_card", None),
+ ("unknown pipeline", "GET", "/api/v1/pipelines/nope/config", None),
+ ("unknown endpoint", "GET", "/api/v1/endpoints/nope/info", None),
+ ("unknown task", "GET", "/api/v1/tasks/00000000-0000-0000-0000-000000000000", None),
+ ):
+ resp = self._req(method, path, json_body=body)
+ if resp is None:
+ self.report.add(g, name, False, "transport error")
+ continue
+ ok = resp.status_code in (400, 401, 403, 404, 422)
+ self.report.add(
+ g,
+ name,
+ ok,
+ "" if ok else f"{method} {path} → {resp.status_code} (expected 4xx)",
+ )
+ # Malformed body must be a 422, not a 500.
+ resp = self._req("POST", f"/api/v1/{self.agent}/invoke", json_body={"query": 12345})
+ ok = resp is not None and resp.status_code in (200, 422)
+ self.report.add(
+ g,
+ "typed body validation",
+ ok,
+ "" if ok else f"→ {getattr(resp, 'status_code', 'transport error')}",
+ )
+
+ def verify_logs_history(self) -> None:
+ """Per-agent invocation history, when the deployment records it."""
+ g = "logs-history"
+ base = f"/api/v1/{self.agent}"
+ resp = self._req("GET", f"{base}/logs?limit=5")
+ if resp is None:
+ self.report.add(g, "GET logs", False, "transport error")
+ return
+ if resp.status_code == 400:
+ self.report.skip(g, "logs history", "disabled (AGENTOMATIC_LOGS_HISTORY=0)")
+ return
+
+ def _logs(p: Any) -> str:
+ entries = p.get("logs") if isinstance(p, dict) else p
+ return "" if isinstance(entries, list) else f"no logs list: {p!r}"
+
+ payload = self.check(g, "GET logs", "GET", f"{base}/logs?limit=5", validate=_logs)
+ entries = (payload or {}).get("logs") if isinstance(payload, dict) else None
+ # This deployment has served traffic already, so history must be non-empty.
+ ok = bool(entries)
+ self.report.add(
+ g, "history records invocations", ok, "" if ok else "no entries after traffic"
+ )
+ if entries:
+ log_id = entries[0].get("id")
+ if log_id:
+ self.check(g, "GET log by id", "GET", f"{base}/logs/{log_id}")
+
+ # LLM analysis over those logs is opt-in and must say so when off.
+ resp = self._req("GET", f"{base}/logs/analysis")
+ if resp is not None and resp.status_code == 400:
+ body = resp.text.lower()
+ ok = "allow_logsllm_analysis" in body or "disabled" in body
+ self.report.add(
+ g,
+ "log analysis refuses clearly when disabled",
+ ok,
+ "" if ok else f"unhelpful 400 body: {resp.text[:160]}",
+ )
+ else:
+ self.check(g, "GET log analysis", "GET", f"{base}/logs/analysis")
+
+ def verify_optimize(self) -> None:
+ """The optimize-aware invoke path."""
+ g = "optimize"
+ self.check(
+ g,
+ "POST optimize/invoke",
+ "POST",
+ f"/api/v1/{self.agent}/optimize/invoke",
+ json_body={"query": "optimize e2e"},
+ )
+ self.check(
+ g,
+ "GET optimization-runs",
+ "GET",
+ f"/api/v1/{self.agent}/optimization-runs",
+ expect=(200, 400),
+ )
+
+ def verify_rate_limit(self) -> None:
+ """Rate limiting, when the deployment enables it.
+
+ Two properties matter in production and they pull in opposite
+ directions: user traffic must be limited, and the endpoints that keep
+ a pod alive — liveness/readiness probes and the metrics scrape — must
+ never be. A kubelet and a scraper share one source IP with real
+ traffic behind a NAT or ingress, so a limiter that counts them will
+ restart healthy pods under load.
+ """
+ g = "rate-limit"
+ resp = self._req("GET", "/api/v1/agents")
+ if resp is None:
+ self.report.add(g, "rate limit probe", False, "transport error")
+ return
+ if "X-RateLimit-Limit" not in resp.headers:
+ self.report.skip(g, "rate limiting", "not enabled on this deployment")
+ return
+ limit = int(resp.headers.get("X-RateLimit-Limit") or 0)
+ self.report.add(g, "advertises X-RateLimit-Limit", limit > 0, f"limit={limit}")
+
+ # Probes and the scrape must survive well past the budget.
+ for path in ("/health", "/ready", "/readiness"):
+ codes = set()
+ for _ in range(limit + 20):
+ got = self._req("GET", path, honour_retry_after=False)
+ codes.add(getattr(got, "status_code", "transport error"))
+ ok = 429 not in codes
+ self.report.add(g, f"{path} never throttled", ok, "" if ok else f"codes={codes}")
+ codes = set()
+ for _ in range(limit + 20):
+ got = self._req("GET", "/metrics", honour_retry_after=False)
+ codes.add(getattr(got, "status_code", "transport error"))
+ ok = 429 not in codes
+ self.report.add(g, "/metrics never throttled", ok, "" if ok else f"codes={codes}")
+
+ # …and that flood must not have spent the caller's budget. Compare the
+ # advertised remaining count either side of it rather than just calling
+ # a route: by this point the harness has legitimately used budget of its
+ # own, so "a normal call still works" would be measuring the wrong thing.
+ before = self._remaining_budget()
+ for path in ("/health", "/ready", "/readiness", "/metrics"):
+ for _ in range(10):
+ self._req("GET", path, honour_retry_after=False)
+ after = self._remaining_budget()
+ if before is None or after is None:
+ self.report.skip(g, "probes do not consume the user budget", "no budget header")
+ else:
+ # `after` is read with one billed request, so allow that single unit.
+ ok = after >= before - 1
+ self.report.add(
+ g,
+ "probes do not consume the user budget",
+ ok,
+ "" if ok else f"remaining fell {before} → {after} across 40 probes",
+ )
+
+ def _remaining_budget(self) -> int | None:
+ """Return the limiter's advertised remaining requests, if it advertises one."""
+ resp = self._req("GET", "/api/v1/agents")
+ if resp is None or "X-RateLimit-Remaining" not in resp.headers:
+ return None
+ try:
+ return int(resp.headers["X-RateLimit-Remaining"])
+ except ValueError:
+ return None
+
+ def run_all(self) -> Report:
+ """Run every group and return the report."""
+ self.verify_platform()
+ self.verify_studio()
+ self.verify_agent_rest()
+ self.verify_a2a()
+ self.verify_plugins()
+ self.verify_endpoints()
+ self.verify_ingestion()
+ self.verify_pipelines()
+ self.verify_every_pipeline()
+ self.verify_isolation()
+ self.verify_tasks()
+ self.verify_logs_history()
+ self.verify_optimize()
+ self.verify_metrics()
+ self.verify_rate_limit()
+ self.verify_auth()
+ self.verify_error_contract()
+ # Control plane last: it toggles agent availability.
+ self.verify_control_plane()
+ return self.report
+
+ def close(self) -> None:
+ """Release the HTTP client."""
+ self.client.close()
+
+
+def main() -> int:
+ """Parse arguments, run the suite, print the report."""
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("--base-url", default="http://127.0.0.1:8000")
+ ap.add_argument("--api-key", default="")
+ ap.add_argument("--control-token", default="")
+ ap.add_argument("--agent", default="ag_basic")
+ ap.add_argument("--plugin", default="scorer")
+ ap.add_argument("--pipeline", default="basic_flow")
+ ap.add_argument("--endpoint", default="echo")
+ ap.add_argument("--ingestor", default="")
+ ap.add_argument("--timeout", type=float, default=30.0)
+ ap.add_argument("--expect-auth", action="store_true")
+ ap.add_argument("--no-studio", action="store_true")
+ ap.add_argument("--json", default="")
+ ap.add_argument(
+ "--wait",
+ type=float,
+ default=0.0,
+ help="Seconds to wait for the server to answer /health before starting.",
+ )
+ args = ap.parse_args()
+
+ if args.wait:
+ deadline = time.time() + args.wait
+ while time.time() < deadline:
+ try:
+ r = httpx.get(f"{args.base_url.rstrip('/')}/health", timeout=5.0)
+ if r.status_code == 200:
+ break
+ except Exception: # noqa: BLE001
+ pass
+ time.sleep(1.0)
+
+ v = Verifier(
+ base_url=args.base_url,
+ api_key=args.api_key,
+ control_token=args.control_token,
+ agent=args.agent,
+ plugin=args.plugin,
+ pipeline=args.pipeline,
+ endpoint=args.endpoint,
+ ingestor=args.ingestor,
+ timeout=args.timeout,
+ expect_auth=args.expect_auth,
+ expect_studio=not args.no_studio,
+ )
+ try:
+ report = v.run_all()
+ finally:
+ v.close()
+
+ print(report.render())
+ if args.json:
+ with open(args.json, "w") as fh:
+ json.dump(
+ [
+ {
+ "group": c.group,
+ "name": c.name,
+ "ok": c.ok,
+ "skipped": c.skipped,
+ "detail": c.detail,
+ }
+ for c in report.checks
+ ],
+ fh,
+ indent=2,
+ )
+ return 1 if report.failures else 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/keras_showcase.py b/scripts/keras_showcase.py
new file mode 100644
index 0000000..66fc6c1
--- /dev/null
+++ b/scripts/keras_showcase.py
@@ -0,0 +1,295 @@
+"""Run the Keras-style agent lifecycle end to end against a real model.
+
+``compile() -> fit() -> evaluate() -> save() -> load()``, with a real
+optimizer driving a real LLM, a real metric, and an ``EarlyStopping``
+callback. Every number printed is measured: the loss curve comes from the
+returned ``History``, and the before/after scores from ``evaluate()``.
+
+Point it at any OpenAI-compatible endpoint -- a local oMLX / llama.cpp /
+vLLM / LM Studio server, or a hosted one::
+
+ export OMLX_BASE_URL=http://127.0.0.1:8000/v1
+ export OMLX_API_KEY=whatever
+ python scripts/keras_showcase.py --model omlx/my-local-model
+
+The agent here is deliberately simple and deterministic: its answers improve
+only when the system prompt contains a specific token it must *discover* from
+its own failures. That makes an improvement in the curve attributable to the
+optimizer rather than to model variance -- which is the point of a showcase.
+
+Exit code is ``0`` only when every check passes.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+import tempfile
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from agentomatic.agents.base import BaseGraphAgent
+from agentomatic.agents.history import EarlyStopping
+from agentomatic.agents.optimizers import PromptFitterBridge
+from agentomatic.agents.types import AgentDataset, AgentExample
+from agentomatic.optimize import PromptSearchSpace
+from agentomatic.optimize.metrics import ContainsMetric
+
+#: The token the optimizer has to discover. Present in the prompt, the agent
+#: answers well; absent, it does not.
+TARGET = "banana"
+BASELINE_PROMPT = "Answer the question."
+
+failures: list[str] = []
+
+
+def check(label: str, ok: bool, detail: str = "") -> None:
+ """Record and print one assertion."""
+ print(f" {'PASS' if ok else 'FAIL'} {label}{(' — ' + detail) if detail else ''}")
+ if not ok:
+ failures.append(f"{label}: {detail}")
+
+
+@dataclass
+class ShowcaseState:
+ """Per-run state for :class:`ShowcaseAgent`."""
+
+ request: str = ""
+ output: dict[str, Any] = field(default_factory=dict)
+
+
+class ShowcaseAgent(BaseGraphAgent[ShowcaseState]):
+ """Answers well only once the prompt carries the target token."""
+
+ agent_name = "keras_showcase"
+ system_prompt = BASELINE_PROMPT
+
+ def build_graph(self) -> Any:
+ """Single-node graph."""
+ g = self.new_graph()
+ g.add_node("respond", self.respond)
+ g.set_entry_point("respond")
+ g.set_finish_point("respond")
+ return g.compile()
+
+ def respond(self, state: ShowcaseState) -> ShowcaseState:
+ """Answer, conditioned on whether the prompt was optimized."""
+ prompt = self.resolve_system_prompt(default=self.system_prompt)
+ improved = TARGET in prompt.lower()
+ marker = "OPT" if improved else "BASE"
+ state.output = {
+ "response": f"{marker}: answer to {state.request}"
+ + (f" {TARGET}" if improved else ""),
+ "used_prompt": prompt,
+ }
+ return state
+
+ def input_to_state(self, input_data: dict[str, Any]) -> ShowcaseState:
+ """Accept both the REST (`current_query`) and inline (`query`) forms."""
+ return ShowcaseState(
+ request=input_data.get("current_query") or input_data.get("query") or ""
+ )
+
+ def state_to_output(self, state: ShowcaseState) -> dict[str, Any]:
+ """Publish the agent's answer."""
+ return state.output
+
+
+class TargetMetric:
+ """1.0 when the answer carries the target token, else 0.0."""
+
+ name = "target_hit"
+
+ def score(self, example: AgentExample, prediction: dict[str, Any]) -> float:
+ """Score one prediction."""
+ del example
+ return 1.0 if TARGET in str(prediction.get("response", "")).lower() else 0.0
+
+
+def build_dataset() -> AgentDataset:
+ """A dataset with explicit train / validation / test splits."""
+ queries = [
+ "capital of france",
+ "2 + 2",
+ "colour of the sky",
+ "hello world",
+ "largest ocean",
+ "who wrote hamlet",
+ "speed of light",
+ "tallest mountain",
+ ]
+ # The fitter warns below 4 train / 2 validation, and a warned-about run is
+ # not a showcase.
+ splits = [
+ "train",
+ "train",
+ "train",
+ "train",
+ "validation",
+ "validation",
+ "test",
+ "test",
+ ]
+ return AgentDataset(
+ name="keras_showcase",
+ examples=[
+ AgentExample(
+ id=f"e{i}",
+ input={"current_query": q},
+ # Comma-separated keywords, read by the optimizer's metric.
+ expected_output={"response": f"OPT, {TARGET}"},
+ split=split,
+ metadata={"split": split},
+ )
+ for i, (q, split) in enumerate(zip(queries, splits, strict=True))
+ ],
+ )
+
+
+def main() -> int:
+ """Run the lifecycle and report."""
+ ap = argparse.ArgumentParser(
+ description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+ )
+ ap.add_argument(
+ "--model",
+ default=os.environ.get("AGENTOMATIC_LIVE_MODEL", ""),
+ help="e.g. omlx/my-model, ollama/qwen2.5:7b, openai/gpt-4o-mini",
+ )
+ ap.add_argument("--base-url", default=os.environ.get("OMLX_BASE_URL", ""))
+ ap.add_argument("--api-key", default=os.environ.get("OMLX_API_KEY", ""))
+ ap.add_argument("--epochs", type=int, default=2)
+ ap.add_argument("--max-trials", type=int, default=6)
+ args = ap.parse_args()
+
+ if not args.model:
+ print("No model given. Pass --model (or set AGENTOMATIC_LIVE_MODEL).")
+ print("The optimizer needs a real LLM — there is nothing to showcase without one.")
+ return 2
+
+ ds = build_dataset()
+ agent = ShowcaseAgent()
+ metric = TargetMetric()
+
+ print(f"── compile() — model={args.model} ───────────────────────")
+ bridge = PromptFitterBridge(
+ agent_name=agent.agent_name,
+ task_model=args.model,
+ rewrite_model=args.model,
+ optimizer="rewrite",
+ # The dataset's expected_output is comma-separated keywords, so the
+ # fit objective has to be keyword containment. The bridge defaults to
+ # ExactMatchMetric, which never fires here and leaves the curve flat.
+ metric=ContainsMetric(),
+ max_trials=args.max_trials,
+ search_space=PromptSearchSpace(
+ optimize_system_prompt=True,
+ optimize_model_params=False,
+ optimize_few_shot=False,
+ ),
+ auto_report=False,
+ concurrency=1,
+ llm_base_url=args.base_url or None,
+ llm_api_key=args.api_key or None,
+ )
+ agent.compile(ds, metrics=[metric], optimizer=bridge, loss=metric)
+ meta = agent.compiled_metadata
+ check("dataset recorded", meta.get("dataset_size") == len(ds), str(meta))
+ check("metric recorded", metric.name in (meta.get("metrics") or []), str(meta))
+ check("optimizer recorded", meta.get("optimizer") == "PromptFitterBridge", str(meta))
+ check("loss recorded", meta.get("loss") not in (None, "none"), str(meta))
+
+ print("\n── evaluate() before fit ────────────────────────────────")
+ before = agent.evaluate(ds).scores[metric.name]
+ print(f" {metric.name} = {before:.3f}")
+ check("baseline scored every example", len(agent.evaluate(ds).example_results) == len(ds))
+
+ print(f"\n── fit(epochs={args.epochs}) with EarlyStopping ─────────")
+ history = agent.fit(
+ ds,
+ epochs=args.epochs,
+ verbose=0,
+ optimize_mode="rewrite",
+ max_trials=args.max_trials,
+ callbacks=[EarlyStopping(monitor="loss", patience=2)],
+ )
+ curve = list(history.history.get("loss", []))
+ scores = list(history.history.get(metric.name, []))
+ print(f" loss: {[round(x, 3) for x in curve]}")
+ print(f" {metric.name:14s} {[round(x, 3) for x in scores]}")
+ check("fit returned a loss curve", bool(curve), str(history.history))
+ check("fit tracked the compiled metric", bool(scores), str(history.history))
+ if len(curve) > 1:
+ check(
+ "loss never increased",
+ curve[-1] <= curve[0] + 1e-9,
+ f"{curve[0]:.3f} -> {curve[-1]:.3f}",
+ )
+
+ status = getattr(agent, "_last_optimize_status", None)
+ check("the optimizer ran rather than silently skipping", status == "ok", str(status))
+
+ result = getattr(agent, "_last_fit_result", None)
+ check("fit left an auditable result", result is not None)
+ if result is not None:
+ print(
+ f" baseline={result.baseline_score:.4f} best={result.best_score:.4f}"
+ f" holdout={result.holdout_score}"
+ )
+ improved = result.best_score > result.baseline_score + 1e-9
+ if improved:
+ tuned = agent.compiled_config.get("system_prompt") or ""
+ check("the tuned prompt found the target token", TARGET in tuned.lower(), tuned[:80])
+ else:
+ # No improvement is a legitimate outcome — a silent one is not.
+ trail = len(result.prompt_history or []) + len(result.trials or [])
+ check(
+ "no-improvement still left an auditable trail",
+ trail > 0,
+ f"{trail} prompt/trial record(s)",
+ )
+
+ print("\n── evaluate() after fit ─────────────────────────────────")
+ after = agent.evaluate(ds).scores[metric.name]
+ print(f" {metric.name} = {after:.3f} (was {before:.3f})")
+ check("score did not regress", after >= before - 1e-9, f"{before:.3f} -> {after:.3f}")
+
+ print("\n── save() / load() round trip ───────────────────────────")
+ with tempfile.TemporaryDirectory() as tmp:
+ path = Path(tmp) / "showcase.agent"
+ agent.save(path)
+ written = sorted(f.name for f in path.iterdir())
+ check("save wrote the compiled config", "config.json" in written, str(written))
+ check("save wrote the fit history", "fit_history.json" in written, str(written))
+
+ restored = ShowcaseAgent()
+ restored.load(path)
+ check(
+ "the tuned prompt survived the round trip",
+ restored.resolve_system_prompt(default="") == agent.resolve_system_prompt(default=""),
+ )
+ check("the fit history survived", bool(restored.history.history.get("loss")))
+
+ # Metrics are live objects and never serialise — supply them again.
+ reloaded = restored.evaluate(ds, metrics=[metric]).scores[metric.name]
+ print(f" reloaded {metric.name} = {reloaded:.3f}")
+ check(
+ "a reloaded agent scores identically",
+ abs(reloaded - after) < 1e-9,
+ f"{after:.3f} vs {reloaded:.3f}",
+ )
+
+ print()
+ if failures:
+ print(f"KERAS LIFECYCLE FAILED — {len(failures)} check(s)")
+ for failure in failures:
+ print(f" - {failure}")
+ return 1
+ print("KERAS LIFECYCLE PASSED — compile / fit / evaluate / save / load on a real model")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/local_slm_server.py b/scripts/local_slm_server.py
new file mode 100644
index 0000000..48468d9
--- /dev/null
+++ b/scripts/local_slm_server.py
@@ -0,0 +1,499 @@
+"""A local OpenAI-compatible server that stands in for a small instruct model.
+
+**This is a test double, not a language model.** It generates nothing; it
+follows a fixed set of rules. Point the live optimization suites at a real
+local model (oMLX, llama.cpp, vLLM, LM Studio, Ollama) whenever you have one —
+they speak the same protocol and need no changes here::
+
+ export OMLX_BASE_URL=http://127.0.0.1:8000/v1
+ export OMLX_API_KEY=…
+ export AGENTOMATIC_LIVE_MODEL=omlx/your-model
+ uv run pytest tests/test_live_omlx_optimize.py tests/test_live_omlx_keras_optimize.py \
+ -q --override-ini='addopts='
+
+Why it exists: those suites skip entirely when no OpenAI-compatible endpoint
+is reachable, which leaves the whole ``omlx/`` provider path, the prompt
+fitter, and the Keras-style ``fit()`` loop unexercised on any machine without
+a local model — including CI. Running them against this server exercises all
+of it deterministically.
+
+What makes it a valid optimization *target* rather than a constant function:
+answer quality genuinely depends on the system prompt. Each directive a prompt
+carries ("answer as JSON", "cite your source", "state your confidence", or a
+required marker token) makes the response satisfy one more property the metric
+rewards, so a better prompt scores strictly higher. An optimizer that really
+searches and selects will climb; one that does not, will not.
+
+It also plays the two other roles the fitter needs. As the **rewriter** it
+reads the optimization briefing — current prompt, failing I/O, expected
+answers, judge guidance — and folds the missing required tokens into a new
+prompt, which is the move a real rewrite model makes. As the **judge** it
+returns the exact schema the metric asked for, scoring the same properties the
+metric does so the two signals agree.
+
+What it cannot tell you: whether a real model writes *good* prompts. It proves
+the machinery — search, evaluation, selection, early stopping, checkpointing,
+config application — not the quality of a real model's language.
+
+Run it::
+
+ uv run python scripts/local_slm_server.py --port 8000
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+import time
+from collections import deque
+from typing import Any
+
+from fastapi import FastAPI, Request
+from fastapi.responses import JSONResponse
+
+app = FastAPI(title="stub-slm", version="1.0.0")
+
+MODEL_ID = "stub-slm-1b-instruct"
+
+#: Directives the responder understands, and the behaviour each unlocks.
+DIRECTIVES: dict[str, tuple[str, ...]] = {
+ "json": ("json", "structured output", "as json", "json object"),
+ "cite": ("cite", "citation", "source", "reference"),
+ "concise": ("concise", "brief", "short", "succinct"),
+ "steps": ("step by step", "step-by-step", "reasoning", "explain how"),
+ "confidence": ("confidence", "certainty", "how sure"),
+}
+
+#: Ground-truth answers the responder knows, keyed by a term in the question.
+FACTS: dict[str, str] = {
+ "capital of france": "Paris",
+ "capital of japan": "Tokyo",
+ "capital of brazil": "Brasilia",
+ "largest ocean": "the Pacific Ocean",
+ "speed of light": "299792458 metres per second",
+ "boiling point": "100 degrees Celsius at sea level",
+}
+
+
+def _seed(text: str) -> int:
+ """Deterministic seed so identical requests give identical answers."""
+ return int(hashlib.sha256(text.encode()).hexdigest()[:8], 16)
+
+
+def _active_directives(system_prompt: str) -> set[str]:
+ """Return the directive names the system prompt asks for."""
+ low = system_prompt.lower()
+ return {name for name, words in DIRECTIVES.items() if any(w in low for w in words)}
+
+
+def _lookup(question: str) -> str | None:
+ """Return the known answer for *question*, if the responder knows one."""
+ low = question.lower()
+ for key, answer in FACTS.items():
+ if key in low:
+ return answer
+ return None
+
+
+#: Words too common to be a meaningful optimization target.
+_STOPWORDS = frozenset(
+ """a an the and or of to in on for with is are be as at by from that this it
+ answer response output result query question expected actual""".split()
+)
+
+
+def _expected_keywords(briefing: str) -> list[str]:
+ """Mine the required tokens out of an optimization briefing.
+
+ A briefing states ground truth in several places, and a competent rewrite
+ model reads all of them:
+
+ * ``Expected:`` / ``## Expected answer`` — the target answer itself.
+ * Judge guidance naming marker tokens the ideal answer contains.
+ * Metric feedback of the form ``missing the required marker token 'x'``.
+
+ Args:
+ briefing: The full rewrite or mutation prompt sent by the fitter.
+
+ Returns:
+ Distinctive required tokens, most frequently demanded first.
+ """
+ sources: list[str] = []
+ sources += re.findall(r"^\s*[-*]?\s*Expected(?:\s+answer)?:\s*(.+)$", briefing, re.MULTILINE)
+ # ``to_datapoint`` renders the answer as its own markdown block, which the
+ # briefing indents under the label. Allow leading whitespace on both the
+ # header and the value, or the ground truth is missed entirely and the
+ # rewrite has nothing to fold in.
+ sources += re.findall(r"^[ \t]*##\s*Expected answer\s*\n[ \t]*(.+)$", briefing, re.MULTILINE)
+
+ counts: dict[str, int] = {}
+ for line in sources:
+ for token in re.split(r"[\s,;]+", line.strip()):
+ word = token.strip("\"'`.:()[]{}").strip()
+ if len(word) < 2 or word.lower() in _STOPWORDS or word.startswith("#"):
+ continue
+ counts[word] = counts.get(word, 0) + 3 # ground truth outranks hints
+
+ # Tokens the guidance or the metric feedback names explicitly.
+ for quoted in re.findall(
+ r"(?:marker token[s]?|required token[s]?|keyword[s]?)[^\n]*?((?:'[^']+'|\"[^\"]+\")(?:[^\n]*?(?:'[^']+'|\"[^\"]+\"))*)",
+ briefing,
+ re.IGNORECASE,
+ ):
+ for word in re.findall(r"['\"]([^'\"]+)['\"]", quoted):
+ word = word.strip()
+ if len(word) < 2 or word.lower() in _STOPWORDS:
+ continue
+ counts[word] = counts.get(word, 0) + 2
+ return [w for w, _ in sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))]
+
+
+def _current_prompt(briefing: str) -> str:
+ """Return the prompt the briefing is asking us to improve."""
+ for header in ("### System prompt", "## Draft system prompt", "## Draft"):
+ idx = briefing.find(header)
+ if idx == -1:
+ continue
+ block = re.search(r"```(?:\w+)?\n(.*?)```", briefing[idx:], re.DOTALL)
+ if block:
+ return block.group(1).strip()
+ return ""
+
+
+def _pass_kind(briefing: str) -> str:
+ """Return DRAFT / CRITIQUE / REVISE for a multi-pass rewrite briefing."""
+ match = re.search(r"##\s*Task\s*\(pass\s*\d+/\d+:\s*(\w+)\)", briefing, re.IGNORECASE)
+ return match.group(1).upper() if match else "DRAFT"
+
+
+def _rewrite_prompt(user_prompt: str) -> str:
+ """Play the prompt-rewriter across the fitter's DRAFT/CRITIQUE/REVISE passes.
+
+ The fitter hands over a briefing containing the current prompt, the
+ failing I/O, and the expected answers. The move a real rewrite model makes
+ is to fold the missing expected tokens into the prompt while keeping the
+ role intact — so that is what happens here, deterministically.
+ """
+ kind = _pass_kind(user_prompt)
+ keywords = _expected_keywords(user_prompt)
+ base = _current_prompt(user_prompt) or "You are a helpful assistant."
+ # Keep the role sentence, drop any requirement clause a previous pass added.
+ role = base.split("Always include")[0].strip().rstrip(".")
+
+ if kind == "CRITIQUE":
+ gaps = [
+ f"- The prompt never mentions the required token '{k}', which every "
+ f"expected answer contains."
+ for k in keywords[:3]
+ ]
+ gaps.append("- The output format is not stated explicitly.")
+ return "\n".join(gaps)
+
+ clauses: list[str] = []
+ if keywords:
+ required = ", ".join(f"'{k}'" for k in keywords[:4])
+ clauses.append(
+ f"Always include {required} in your answer, exactly as written, for every query."
+ )
+ # Judge feedback and improvement hints in the briefing say what the grader
+ # is actually rewarding. A rewrite model that ignored them would never
+ # climb a rubric-based metric, so act on the ones we recognise.
+ clauses.extend(_hinted_clauses(user_prompt))
+ if not clauses:
+ return f"---\n{role}."
+ return "---\n" + " ".join([f"{role}.", *clauses])
+
+
+#: Instructions the responder honours, and the words a briefing uses to ask
+#: for them — in judge feedback, improvement hints, or the criteria itself.
+_HINT_CLAUSES: tuple[tuple[tuple[str, ...], str], ...] = (
+ (("json", "structured output"), "Always answer as a JSON object."),
+ (("source", "cite", "attribution"), "Always cite your source."),
+ (("confidence", "certainty"), "Always state your confidence."),
+ (("reasoning", "step by step", "step-by-step"), "Explain your reasoning step by step."),
+)
+
+
+def _hinted_clauses(briefing: str) -> list[str]:
+ """Return instructions the briefing's feedback is asking for.
+
+ Args:
+ briefing: The full rewrite prompt, including judge feedback and hints.
+
+ Returns:
+ Instruction sentences to fold into the improved prompt.
+ """
+ # Only read the parts of the briefing that describe what is *wanted*, so a
+ # token appearing in an agent's own output does not look like a request.
+ wanted = "\n".join(
+ line
+ for line in briefing.splitlines()
+ if any(
+ marker in line.lower()
+ for marker in ("hint", "feedback", "criteria", "judge", "what_failed", "missing")
+ )
+ ).lower()
+ return [clause for words, clause in _HINT_CLAUSES if any(w in wanted for w in words)]
+
+
+def _requested_dimensions(prompt: str) -> list[str]:
+ """Return the dimension names a judge prompt asks to be scored."""
+ block = re.search(r'"dimensions"\s*:\s*\{(.*?)\}', prompt, re.DOTALL)
+ if not block:
+ return []
+ return re.findall(r'"(\w+)"\s*:', block.group(1))
+
+
+def _judge_payload(prompt: str) -> str:
+ """Grade a candidate answer in the exact schema the judge asked for.
+
+ Structure and attribution are what the metric rewards, so the judge
+ rewards them too — an LLM-as-judge that disagreed with the metric would
+ give the optimizer a contradictory signal.
+ """
+ low = prompt.lower()
+ score = 0.2
+ for marker, weight in (("source", 0.25), ("confidence", 0.2), ("{", 0.25), ("reasoning", 0.1)):
+ if marker in low:
+ score += weight
+ score = min(1.0, round(score, 3))
+ payload: dict[str, Any] = {
+ "overall_score": score,
+ "score": score,
+ "feedback": "Graded on structure, attribution and stated confidence.",
+ "motivation": (
+ "The response was checked for a structured body, an explicit source "
+ "and a stated confidence; each present raises the score."
+ ),
+ "what_worked": ["structured output"] if "{" in low else [],
+ "what_failed": [] if score > 0.6 else ["missing attribution or structure"],
+ "improvement_hints": (
+ [] if score > 0.6 else ["Require JSON output and an explicit source in the prompt."]
+ ),
+ }
+ dims = _requested_dimensions(prompt)
+ if dims:
+ payload["dimensions"] = {d: score for d in dims}
+ return json.dumps(payload)
+
+
+def _literal_echo(prompt: str) -> str | None:
+ """Honour 'reply with exactly X' instructions, as an instruct model would."""
+ match = re.search(
+ r"repl(?:y|ies)\s+with\s+exactly\s+(?:the\s+word\s+)?[\"'`]?([\w .-]{1,40}?)[\"'`]?"
+ r"(?:\s+and\s+nothing\s+else)?[.!]?\s*$",
+ prompt.strip(),
+ re.IGNORECASE,
+ )
+ return match.group(1).strip() if match else None
+
+
+def _echo_requested_json(prompt: str) -> str | None:
+ """Return the JSON object a prompt literally asks to be returned."""
+ match = re.search(r"[Rr]eturn\s+JSON\s*(\{.*?\})", prompt, re.DOTALL)
+ if not match:
+ return None
+ try:
+ return json.dumps(json.loads(match.group(1)))
+ except Exception: # noqa: BLE001 - not a literal object, fall through
+ return None
+
+
+def _is_rewrite_request(prompt: str) -> bool:
+ """Return whether the caller is asking for an improved prompt."""
+ low = prompt.lower()
+ return any(
+ w in low
+ for w in (
+ "rewrite",
+ "improved prompt",
+ "improved system prompt",
+ "new system prompt",
+ "better prompt",
+ "propose a prompt",
+ "draft system prompt",
+ "optimization briefing",
+ "mutated system prompt",
+ "prompt mutation",
+ )
+ )
+
+
+def _is_judge_request(prompt: str) -> bool:
+ """Return whether the caller is asking for a graded evaluation.
+
+ Keyed on ``overall_score``, which only the judge schema asks for. Looser
+ words do not work: a judge prompt talks about "the prompt rewriter", so
+ matching on "rewrite" sends grading requests down the rewrite path.
+ """
+ low = prompt.lower()
+ return "overall_score" in low or "evaluation judge" in low or "dimensions to score" in low
+
+
+def _answer(system_prompt: str, question: str) -> str:
+ """Answer *question* honouring whatever the system prompt asked for.
+
+ This is the behaviour that makes the server a real optimization target:
+ every directive the prompt carries adds a property the metric rewards, so
+ a better prompt produces a strictly better answer.
+ """
+ directives = _active_directives(system_prompt)
+ fact = _lookup(question)
+ core = fact if fact else "I don't have a definitive answer for that."
+
+ if "json" in directives:
+ payload: dict[str, Any] = {"answer": core}
+ if "cite" in directives:
+ payload["source"] = "internal-knowledge-base"
+ if "confidence" in directives:
+ payload["confidence"] = 0.92 if fact else 0.20
+ if "steps" in directives:
+ payload["reasoning"] = "Matched the question against known facts."
+ return json.dumps(payload)
+
+ parts = [core]
+ if "steps" in directives:
+ parts.append("Reasoning: matched the question against known facts.")
+ if "cite" in directives:
+ parts.append("Source: internal-knowledge-base.")
+ if "confidence" in directives:
+ parts.append(f"Confidence: {0.92 if fact else 0.20}.")
+ text = " ".join(parts)
+ if "concise" in directives:
+ text = parts[0] if len(parts) == 1 else " ".join(parts[:2])
+ return text
+
+
+@app.get("/v1/models")
+async def list_models() -> dict[str, Any]:
+ """OpenAI-compatible model listing."""
+ return {
+ "object": "list",
+ "data": [{"id": MODEL_ID, "object": "model", "owned_by": "local"}],
+ }
+
+
+#: The last few request bodies, newest last.
+#:
+#: A verifier can assert what the platform actually put on the wire --
+#: whether conversation history reached the model, whether a system prompt
+#: was sent as a system role -- instead of inferring it from the answer.
+_RECORDED: deque[dict[str, Any]] = deque(maxlen=64)
+
+
+@app.get("/health")
+async def health() -> dict[str, str]:
+ """Liveness probe."""
+ return {"status": "ok", "model": MODEL_ID}
+
+
+@app.get("/debug/requests")
+async def recorded_requests() -> dict[str, Any]:
+ """Return the recent request bodies this double was sent."""
+ return {"count": len(_RECORDED), "requests": list(_RECORDED)}
+
+
+@app.delete("/debug/requests")
+async def clear_recorded_requests() -> dict[str, Any]:
+ """Drop the recorded requests, so a check starts from a clean slate."""
+ _RECORDED.clear()
+ return {"count": 0}
+
+
+@app.post("/v1/chat/completions")
+async def chat_completions(request: Request) -> JSONResponse:
+ """OpenAI-compatible chat completion."""
+ body = await request.json()
+ _RECORDED.append(body)
+ messages = body.get("messages") or []
+ system_prompt = " ".join(
+ str(m.get("content", "")) for m in messages if m.get("role") == "system"
+ )
+ user_prompt = "\n".join(str(m.get("content", "")) for m in messages if m.get("role") == "user")
+
+ # Many agents concatenate their system prompt into the user turn rather
+ # than sending a system role (``llm.invoke(f"{prompt}\n\nUser: {q}")`` is
+ # the shape the scaffolded templates use). A real model reads instructions
+ # wherever they appear, so directives are detected across both turns.
+ instructions = f"{system_prompt}\n{user_prompt}"
+ literal = _literal_echo(user_prompt)
+ echoed = _echo_requested_json(user_prompt)
+ if literal is not None:
+ content = literal
+ elif echoed is not None:
+ content = echoed
+ elif _is_judge_request(user_prompt):
+ content = _judge_payload(user_prompt)
+ elif _is_rewrite_request(user_prompt) or _is_rewrite_request(system_prompt):
+ content = _rewrite_prompt(user_prompt)
+ else:
+ content = _answer(instructions, user_prompt)
+
+ wants_json = (body.get("response_format") or {}).get("type") == "json_object"
+ if wants_json and not content.lstrip().startswith("{"):
+ content = json.dumps({"answer": content})
+
+ prompt_tokens = max(1, len((system_prompt + user_prompt).split()))
+ completion_tokens = max(1, len(content.split()))
+ return JSONResponse(
+ {
+ "id": f"chatcmpl-{_seed(system_prompt + user_prompt):08x}",
+ "object": "chat.completion",
+ "created": int(time.time()),
+ "model": body.get("model") or MODEL_ID,
+ "choices": [
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": content},
+ "finish_reason": "stop",
+ }
+ ],
+ "usage": {
+ "prompt_tokens": prompt_tokens,
+ "completion_tokens": completion_tokens,
+ "total_tokens": prompt_tokens + completion_tokens,
+ },
+ }
+ )
+
+
+@app.post("/v1/completions")
+async def completions(request: Request) -> JSONResponse:
+ """Legacy completion endpoint, for callers that still use it."""
+ body = await request.json()
+ prompt = str(body.get("prompt", ""))
+ literal = _literal_echo(prompt)
+ if literal is not None:
+ content = literal
+ elif _is_rewrite_request(prompt):
+ content = _rewrite_prompt(prompt)
+ else:
+ content = _answer("", prompt)
+ return JSONResponse(
+ {
+ "id": f"cmpl-{_seed(prompt):08x}",
+ "object": "text_completion",
+ "created": int(time.time()),
+ "model": body.get("model") or MODEL_ID,
+ "choices": [{"index": 0, "text": content, "finish_reason": "stop"}],
+ "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
+ }
+ )
+
+
+def main() -> None:
+ """Serve the stand-in model on the requested port."""
+ import argparse
+
+ import uvicorn
+
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--host", default="127.0.0.1")
+ parser.add_argument("--port", type=int, default=8000)
+ args = parser.parse_args()
+ uvicorn.run(app, host=args.host, port=args.port, log_level="info")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/src/agentomatic/agents/base.py b/src/agentomatic/agents/base.py
index b85b4ec..da2a515 100644
--- a/src/agentomatic/agents/base.py
+++ b/src/agentomatic/agents/base.py
@@ -535,6 +535,19 @@ def evaluate(
if metrics is None:
metrics = list(self._compile_metrics)
if not metrics:
+ # A metric is a live Python object, so it cannot be written to
+ # ``config.json`` and does not come back with ``load()``. Say so
+ # when the saved metadata shows which ones were compiled in:
+ # otherwise "compile(metrics=...) first" reads as wrong advice to
+ # somebody who did exactly that before saving.
+ saved = [str(m) for m in (self.compiled_metadata.get("metrics") or [])]
+ if saved:
+ raise ValueError(
+ f"No metrics available. This agent was compiled with "
+ f"{saved} but metrics are live objects and do not survive "
+ "save()/load() — re-supply them with "
+ "compile(metrics=[...]) or evaluate(dataset, metrics=[...])."
+ )
raise ValueError(
"No metrics provided. Pass metrics=... to evaluate() "
"or compile(metrics=...) first."
@@ -1068,6 +1081,11 @@ def save(self, path: str | Path) -> None:
- ``evaluation_history.json`` — past evaluation reports
- ``fit_history.json`` — Keras-style ``History`` from the last ``fit()``
+ Metrics, the loss and the optimizer are live Python objects, so only
+ their *names* are recorded, in ``metadata.json``. A loaded agent can
+ serve and can report its past scores, but needs those objects supplied
+ again — ``compile(metrics=[...])`` — before ``fit()`` or ``evaluate()``.
+
Args:
path: Directory to save to.
"""
@@ -1133,6 +1151,11 @@ def load(self, path: str | Path) -> None:
def load_compiled(self, path: str | Path) -> None:
"""Load compiled config from a saved directory.
+ Restores the tuned configuration, the compile metadata, and both
+ histories. Metrics, loss and optimizer are not restored — they are
+ live objects that were never serialised (see :meth:`save`) — so
+ recompile with them before ``fit()`` or ``evaluate()``.
+
Args:
path: Directory containing saved state.
"""
diff --git a/src/agentomatic/agents/optimizers.py b/src/agentomatic/agents/optimizers.py
index 83032a0..f3ec5c5 100644
--- a/src/agentomatic/agents/optimizers.py
+++ b/src/agentomatic/agents/optimizers.py
@@ -372,6 +372,26 @@ def _build_fitter(self, agent: Any, name: str) -> Any:
"PromptFitterBridge: using compiled system_prompt as baseline ({} chars)",
len(baseline_prompt),
)
+ else:
+ # First epoch: nothing compiled yet. Without this the fitter starts
+ # from its own generic default, so the baseline score measures a
+ # prompt the agent never uses and the rewrite improves on the wrong
+ # text — the author's own ``system_prompt`` silently ignored.
+ resolver = getattr(agent, "resolve_system_prompt", None)
+ if callable(resolver):
+ try:
+ baseline_prompt = resolver(default="") or None
+ except Exception: # noqa: BLE001 - fall through to the attribute
+ baseline_prompt = None
+ if not baseline_prompt:
+ attr = getattr(agent, "system_prompt", None)
+ baseline_prompt = attr if isinstance(attr, str) and attr.strip() else None
+ if baseline_prompt:
+ logger.info(
+ "PromptFitterBridge: using the agent's own system_prompt as "
+ "the baseline ({} chars)",
+ len(baseline_prompt),
+ )
# Same compounding for tuned model params: carry previously-applied
# values (e.g. temperature from a param_search epoch) as the fitter's
diff --git a/src/agentomatic/cli/commands.py b/src/agentomatic/cli/commands.py
index d1b7337..ed17cbd 100644
--- a/src/agentomatic/cli/commands.py
+++ b/src/agentomatic/cli/commands.py
@@ -674,6 +674,15 @@ def run(
from agentomatic import AgentPlatform
+ # ``AGENTOMATIC_*`` is the documented way to configure a deployment, and the
+ # image this repo ships runs `agentomatic run` (not `uvicorn main:app`).
+ # Every switch the scaffolded main.py reads is therefore honoured here too,
+ # so the two entrypoints cannot diverge. Silently dropping them was a
+ # security hazard specifically for the auth switches: a container started
+ # with AGENTOMATIC_ENABLE_AUTH=1 and an API key served an entirely
+ # unauthenticated API while looking correctly configured.
+ env_require_auth = _env_bool("AGENTOMATIC_REQUIRE_AUTH", False)
+ require_auth = require_auth_globally or env_require_auth
kwargs: dict[str, Any] = {
"plugins_dir": plugins_dir,
"endpoints_dir": endpoints_dir,
@@ -686,13 +695,24 @@ def run(
"enable_metrics": _env_bool("AGENTOMATIC_ENABLE_METRICS", True),
"logs_history": _env_bool("AGENTOMATIC_LOGS_HISTORY", False),
"allow_logsllm_analysis": _env_bool("AGENTOMATIC_ALLOW_LOGSLLM_ANALYSIS", False),
+ "enable_auth": _env_bool("AGENTOMATIC_ENABLE_AUTH", False),
+ "auth_api_key": os.getenv("AGENTOMATIC_API_KEY", ""),
+ "enable_jwt_auth": _env_bool("AGENTOMATIC_ENABLE_JWT", require_auth),
+ "enable_zero_trust": _env_bool("AGENTOMATIC_ENABLE_ZERO_TRUST", require_auth),
+ "enable_control_plane": _env_bool("AGENTOMATIC_ENABLE_CONTROL_PLANE", False),
+ "control_token": os.getenv("AGENTOMATIC_CONTROL_TOKEN", ""),
+ "enable_rate_limit": _env_bool("AGENTOMATIC_ENABLE_RATE_LIMIT", False),
+ "rate_limit_trust_proxy_headers": _env_bool(
+ "AGENTOMATIC_RATE_LIMIT_TRUST_PROXY_HEADERS", False
+ ),
}
- if require_auth_globally:
+ if require_auth:
# Auto-enable the zero-trust enforcer so the flag actually has effect.
kwargs["enable_zero_trust"] = True
kwargs["require_auth_globally"] = True
# Without JWT (or API key), every request would be rejected.
- kwargs.setdefault("enable_jwt_auth", True)
+ if not kwargs.get("enable_jwt_auth") and not kwargs.get("auth_api_key"):
+ kwargs["enable_jwt_auth"] = True
# Auto-detect and enable UI
if with_ui:
@@ -1754,6 +1774,7 @@ def optimize(
_run_fitter_optimize(
agent=agent,
dataset=dataset,
+ prompt=prompt,
val_dataset=val_dataset,
test_dataset=test_dataset,
metric_names=[m.strip() for m in metrics.split(",") if m.strip()],
@@ -1840,6 +1861,7 @@ def _run_prompt_only_optimize(
def _run_fitter_optimize(
agent: str,
dataset: str,
+ prompt: str | None,
val_dataset: str | None,
test_dataset: str | None,
metric_names: list[str],
@@ -1940,6 +1962,12 @@ def _load(path: str) -> Any:
"api_base": host,
"auto_report": not no_report,
}
+ if prompt:
+ # --prompt is documented as "overrides prompts.json". It reached the
+ # legacy prompt_only path only, so every fitter mode silently
+ # optimized from the agent's own prompt instead — and reported a
+ # baseline score for a prompt the caller never asked for.
+ fitter_kwargs["baseline_system_prompt"] = prompt
if n_runners is not None:
fitter_kwargs["n_runners"] = n_runners
diff --git a/src/agentomatic/cli/deploy.py b/src/agentomatic/cli/deploy.py
index 1fcc2dc..5361a4a 100644
--- a/src/agentomatic/cli/deploy.py
+++ b/src/agentomatic/cli/deploy.py
@@ -49,6 +49,10 @@
# Project artefacts copied into the image, in a stable order. Only those that
# actually exist in the project are emitted so ``docker build`` never fails on
# a missing COPY source.
+#: uv version baked into generated images. Pinned so a build is reproducible;
+#: an unpinned installer changes image contents with no diff to show for it.
+UV_VERSION = "0.8.17"
+
_PROJECT_COPY_CANDIDATES: tuple[str, ...] = (
"main.py",
"agents",
@@ -58,9 +62,58 @@
"pipelines",
"stacks",
"requirements.txt",
+ "pyproject.toml",
+ "uv.lock",
)
+def _requirements_install(project_root: Path | None, *, target: str = "") -> str:
+ """Return the build step that installs the project's own dependencies.
+
+ ``requirements.txt`` is where a project declares what *it* needs on top of
+ agentomatic — a vendor LLM driver, a vector client, an in-house package.
+ The generated image copied the file and never installed it, so every such
+ dependency was silently absent at runtime: an agent configured for a
+ provider whose driver lived only there could not be built.
+
+ A project that keeps a ``uv.lock`` gets ``uv sync --frozen`` instead, which
+ installs the exact resolved versions the lock pins rather than re-resolving
+ at build time.
+
+ Args:
+ project_root: Project directory, scanned for ``uv.lock`` and
+ ``requirements.txt``. When ``None``, the requirements step is
+ emitted anyway (standalone Dockerfile).
+ target: Optional ``--target=DIR`` for the distroless layout.
+
+ Returns:
+ The ``COPY`` + ``RUN`` pair to install project dependencies, or an
+ empty string when the project declares none.
+ """
+ if (
+ project_root is not None
+ and not target
+ and (project_root / "uv.lock").exists()
+ and (project_root / "pyproject.toml").exists()
+ ):
+ return (
+ "\n# This project pins its full dependency tree in uv.lock, so install\n"
+ "# exactly that rather than re-resolving at build time.\n"
+ "COPY pyproject.toml uv.lock ./\n"
+ "RUN uv sync --frozen --no-dev --inexact\n"
+ )
+ if project_root is not None and not (project_root / "requirements.txt").exists():
+ return ""
+ flag = f"--target={target} " if target else ""
+ return (
+ "\n# The project's own dependencies (vendor LLM drivers, vector clients,\n"
+ "# in-house packages). Installed after agentomatic so the pinned version\n"
+ "# above wins, and skipped entirely when the project declares none.\n"
+ "COPY requirements.txt ./requirements.txt\n"
+ f"RUN uv pip install {flag}-r requirements.txt\n"
+ )
+
+
def _copy_lines(project_root: Path | None, *, chown: str | None = None) -> list[str]:
"""Return ``COPY`` lines for the project artefacts that exist.
@@ -217,6 +270,8 @@ def render_dockerfile(
baked-in ``AGENTOMATIC_*`` env defaults.
"""
copies = "\n".join(_copy_lines(project_root, chown="appuser:appuser"))
+ requirements = _requirements_install(project_root)
+ uv_version = UV_VERSION
profile_env_block = _profile_env_block(profile)
return f"""\
# =============================================================================
@@ -239,13 +294,23 @@ def render_dockerfile(
WORKDIR /app
+# uv resolves and installs an order of magnitude faster than pip, which is
+# most of a container build's wall clock. Pinned so the image is reproducible.
+ARG UV_VERSION={uv_version}
+RUN pip install --no-cache-dir "uv==${{UV_VERSION}}"
+
# Isolated virtualenv so we can copy just the deps into the runtime stage.
RUN python -m venv /app/.venv
-ENV PATH="/app/.venv/bin:$PATH"
-
-RUN pip install --upgrade pip \\
- && pip install "agentomatic[all]=={version}"
+ENV PATH="/app/.venv/bin:$PATH" \\
+ VIRTUAL_ENV="/app/.venv"
+# ``db-postgres`` on top of ``all``: the ``all`` extra deliberately ships
+# only the SQLite driver, but the .env generated next to this Dockerfile
+# wires DB__URL to a DATABASE_URL that is usually Postgres — without the
+# driver that container starts and then fails with "No module named
+# 'asyncpg'" the moment persistence is switched on.
+RUN uv pip install "agentomatic[all,db-postgres]=={version}"
+{requirements}
# ---- Runtime stage ----------------------------------------------------------
FROM python:3.12-slim
@@ -320,6 +385,8 @@ def render_dockerfile_distroless(
baked-in ``AGENTOMATIC_*`` env defaults.
"""
copies = "\n".join(_copy_lines(project_root, chown="65532:65532"))
+ requirements = _requirements_install(project_root, target="/app/deps")
+ uv_version = UV_VERSION
profile_env_block = _profile_env_block(profile)
return f"""\
# =============================================================================
@@ -348,8 +415,15 @@ def render_dockerfile_distroless(
# ``--target`` instead of a virtualenv: the runtime stage runs the distroless
# image's own interpreter, which cannot use a venv built around a different
# Python binary. A plain directory on ``PYTHONPATH`` works with any 3.11.
-RUN pip install --upgrade pip \\
- && pip install --target=/app/deps "agentomatic[all]=={version}"
+# ``db-postgres`` on top of ``all``: the ``all`` extra deliberately ships
+# only the SQLite driver, but the .env generated next to this Dockerfile
+# wires DB__URL to a DATABASE_URL that is usually Postgres — without the
+# driver that container starts and then fails with "No module named
+# 'asyncpg'" the moment persistence is switched on.
+ARG UV_VERSION={uv_version}
+RUN pip install --no-cache-dir "uv==${{UV_VERSION}}"
+RUN uv pip install --target=/app/deps "agentomatic[all,db-postgres]=={version}"
+{requirements}
# Copy project sources into the build stage so they can be chowned + carried
# into the runtime stage (distroless cannot chown at runtime — no shell).
diff --git a/src/agentomatic/cli/templates.py b/src/agentomatic/cli/templates.py
index 7505dce..cf03b77 100644
--- a/src/agentomatic/cli/templates.py
+++ b/src/agentomatic/cli/templates.py
@@ -156,14 +156,29 @@ def build_graph(self):
g.set_finish_point("respond")
return g.compile()
+ def _turns(self, state: {title}State) -> list[Any]:
+ """Build the message list for the model, system prompt first.
+
+ ``/chat`` loads the whole thread into ``state.messages``, ending
+ with the current turn. Sending only ``state.request`` would answer
+ every turn as if it were the first — the caller sees
+ ``history_loaded: 4`` and a model that has forgotten all four.
+ """
+ from agentomatic.langchain_adapter import dict_to_messages
+ from langchain_core.messages import SystemMessage
+
+ turns = dict_to_messages(
+ state.messages if state.messages else {{"current_query": state.request}}
+ )
+ return [SystemMessage(content=self._system_prompt()), *turns]
+
def respond(self, state: {title}State) -> {title}State:
- history_len = len(state.messages)
- prompt = self._system_prompt()
+ # ``state.messages`` ends with the current turn, so the history
+ # behind it is one shorter than the list.
+ history_len = max(0, len(state.messages) - 1)
if self.llm is not None:
try:
- result = self.llm.invoke(
- f"{{prompt}}\\n\\nUser: {{state.request}}"
- )
+ result = self.llm.invoke(self._turns(state))
text = getattr(result, "content", None) or str(result)
except Exception as exc: # noqa: BLE001
text = f"[Turn {{history_len + 1}}] (llm error: {{exc}})"
diff --git a/src/agentomatic/connections/manager.py b/src/agentomatic/connections/manager.py
index e816ff0..0ffa456 100644
--- a/src/agentomatic/connections/manager.py
+++ b/src/agentomatic/connections/manager.py
@@ -30,10 +30,69 @@
VectorConnectionConfig,
)
from agentomatic.connections.vector import VectorConnection
+from agentomatic.endpoints.auth import resolve_env
if TYPE_CHECKING:
from collections.abc import Callable
+
+#: The field naming each config kind's connect target. A connection whose
+#: target is an unresolved ``${ENV}`` placeholder cannot possibly work.
+_TARGET_FIELDS: dict[type, str] = {
+ DatabaseConnectionConfig: "url",
+ HttpConnectionConfig: "base_url",
+ VectorConnectionConfig: "url",
+}
+
+
+def unresolved_env_vars(value: Any) -> list[str]:
+ """Return the ``${VAR}`` names in *value* that the environment does not set.
+
+ Args:
+ value: A config value; only strings can carry placeholders.
+
+ Returns:
+ The names of placeholders that resolve to nothing, in order.
+ """
+ import os
+ import re
+
+ if not isinstance(value, str) or "${" not in value:
+ return []
+ return [
+ name
+ for name in re.findall(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}", value)
+ if not os.environ.get(name)
+ ]
+
+
+def _unconfigured_reason(config: Any) -> str | None:
+ """Explain why *config* cannot connect, or ``None`` if it looks usable.
+
+ Only reports the case that is unambiguously a *missing configuration*
+ rather than a broken one: the connect target is built entirely from
+ ``${ENV}`` placeholders that are unset, so it resolves to an empty string.
+
+ Args:
+ config: A connection config.
+
+ Returns:
+ A message naming the environment variables to set, else ``None``.
+ """
+ field = _TARGET_FIELDS.get(type(config))
+ if field is None:
+ return None
+ raw = getattr(config, field, "")
+ missing = unresolved_env_vars(raw)
+ if not missing:
+ return None
+ if resolve_env(raw).strip():
+ # Partially configured — let the driver report what is actually wrong.
+ return None
+ names = ", ".join(missing)
+ return f"{field} is unset — set {names} to enable it"
+
+
#: Scope name used for connections shared across the whole platform.
PLATFORM_SCOPE = "__platform__"
@@ -230,17 +289,48 @@ def first_for_purpose(self, purpose: ConnectionPurpose | str) -> Connection | No
return conns[0] if conns else None
async def initialize(self) -> None:
- """Initialise all connections in this scope."""
+ """Initialise all connections in this scope.
+
+ A connection whose target is nothing but unset ``${ENV}`` placeholders
+ is *unconfigured*, not broken: the scaffolded ``connections.py``
+ declares several as examples. Those are reported as a warning naming
+ the variables to set, rather than as an error carrying whatever the
+ driver made of an empty string ("Could not parse SQLAlchemy URL from
+ given URL string" told an operator nothing about which variable to
+ provide, and four such errors on every boot train people to skim past
+ the ones that matter).
+ """
for name, conn in self._connections.items():
+ reason = _unconfigured_reason(getattr(conn, "config", None))
+ if reason:
+ logger.warning(f"Connection '{name}' not configured: {reason}")
+ continue
try:
await conn.initialize()
except Exception as exc: # noqa: BLE001
logger.error(f"Failed to initialize connection '{name}': {exc}")
async def health_check(self) -> dict[str, Any]:
- """Aggregate health across all connections."""
+ """Aggregate health across all connections.
+
+ A connection whose target is nothing but unset ``${ENV}`` placeholders
+ reports ``not_configured``, not ``unhealthy``. The distinction is what
+ an operator acts on: unhealthy means a backend this deployment depends
+ on is down; not configured means a feature was never switched on, and
+ the message names the variable that would switch it on.
+ """
results: dict[str, Any] = {}
for name, conn in self._connections.items():
+ config = getattr(conn, "config", None)
+ reason = _unconfigured_reason(config)
+ if reason:
+ results[name] = {
+ "connection": name,
+ "kind": str(getattr(config, "kind", "unknown")),
+ "status": "not_configured",
+ "detail": reason,
+ }
+ continue
try:
results[name] = await conn.health_check()
except Exception as exc: # noqa: BLE001
diff --git a/src/agentomatic/core/platform.py b/src/agentomatic/core/platform.py
index 4be48d2..de476db 100644
--- a/src/agentomatic/core/platform.py
+++ b/src/agentomatic/core/platform.py
@@ -36,6 +36,11 @@
from agentomatic.tasks.store import TaskStore
+def _safe_db_url(url: str) -> str:
+ """Return a database URL with any credentials stripped, for logging."""
+ return url.split("@")[-1] if "@" in url else url
+
+
def _agent_tag(name: str) -> str:
"""Return a human-friendly OpenAPI tag for an agent name.
@@ -840,7 +845,11 @@ def _setting(env_key: str, field: str) -> str:
async def _auto_derive_store_from_connections(self) -> None:
"""Populate ``self._store`` from the first MEMORY connection, if any."""
- from agentomatic.connections.manager import PLATFORM_SCOPE, all_managers
+ from agentomatic.connections.manager import (
+ PLATFORM_SCOPE,
+ _unconfigured_reason,
+ all_managers,
+ )
from agentomatic.connections.models import ConnectionPurpose
from agentomatic.connections.stores import create_store_from_connection
@@ -853,12 +862,41 @@ async def _auto_derive_store_from_connections(self) -> None:
candidate = manager.first_for_purpose(ConnectionPurpose.MEMORY)
if candidate is None:
continue
- try:
- self._store = await create_store_from_connection(candidate)
+ # A connection built from unset ${ENV} placeholders is not a
+ # broken store, it is an absent one — the scaffolded
+ # connections.py ships a MEMORY example. Say which variable would
+ # enable it instead of surfacing whatever the driver made of an
+ # empty URL.
+ reason = _unconfigured_reason(getattr(candidate, "config", None))
+ if reason:
logger.info(
- f"🗄️ Auto-derived store from connection "
- f"'{getattr(candidate, 'name', '?')}' in scope '{scope}'"
+ f"Connection '{getattr(candidate, 'name', '?')}' in scope "
+ f"'{scope}' is not configured ({reason}) — not using it as a store."
)
+ continue
+ try:
+ self._store = await create_store_from_connection(candidate)
+ name = getattr(candidate, "name", "?")
+ database_url = self._resolve_database_url()
+ if database_url:
+ # Both are configured and the connection wins. Say so:
+ # otherwise an operator who set DATABASE_URL to a managed
+ # Postgres reads "store configured" and believes their
+ # threads live there, while they are actually going
+ # wherever this connection points — which for the
+ # scaffolded MEMORY example is a file inside the
+ # container, lost on the next restart.
+ logger.warning(
+ f"🗄️ Store taken from MEMORY connection '{name}' (scope "
+ f"'{scope}'), which OVERRIDES the configured "
+ f"DATABASE_URL ({_safe_db_url(database_url)}). Remove "
+ "the MEMORY connection, or point it at the same "
+ "database, if that is not what you intended."
+ )
+ else:
+ logger.info(
+ f"🗄️ Auto-derived store from connection '{name}' in scope '{scope}'"
+ )
return
except Exception as exc: # noqa: BLE001
logger.warning(
diff --git a/src/agentomatic/core/router_factory.py b/src/agentomatic/core/router_factory.py
index c7a165d..3505cff 100644
--- a/src/agentomatic/core/router_factory.py
+++ b/src/agentomatic/core/router_factory.py
@@ -903,12 +903,21 @@ async def chat(request: AgentChatRequest) -> dict[str, Any]:
When a ``thread_id`` is provided and a thread store is configured,
this endpoint automatically:
- 1. Loads prior conversation history into the agent's message context
- 2. Invokes the agent with full conversational awareness
- 3. Persists both user and assistant messages to the store
+
+ 1. Loads prior conversation history into ``state["messages"]``,
+ ending with the current turn.
+ 2. Invokes the agent with that state.
+ 3. Persists both user and assistant messages to the store.
If the conversation exceeds the configured threshold, older messages
are automatically summarised and compressed.
+
+ Whether the model *sees* that history is up to the agent: the
+ platform supplies ``state["messages"]``, and an agent that reads
+ only ``current_query`` answers every turn as if it were the first.
+ ``history_loaded`` in the response counts what was loaded, not what
+ the agent chose to send. See the conversation-memory section of the
+ agents guide.
"""
agent = _get_agent()
thread_id = request.thread_id or f"thread_{uuid.uuid4().hex[:12]}"
diff --git a/src/agentomatic/langchain_adapter.py b/src/agentomatic/langchain_adapter.py
index b030701..3104733 100644
--- a/src/agentomatic/langchain_adapter.py
+++ b/src/agentomatic/langchain_adapter.py
@@ -342,11 +342,15 @@ def to_jsonable(value: Any) -> Any:
on the generic REST/Studio response path, so returning raw ``BaseMessage``
objects (e.g. ``{"messages": state.messages}``) "just works".
"""
- _base_message_cls: Any
+ # Bind through a separate name: importing *as* the annotated name is a
+ # redefinition, which mypy 2.1 rejects.
+ _base_message_cls: Any = None
try:
- from langchain_core.messages import BaseMessage as _base_message_cls
+ from langchain_core.messages import BaseMessage
+
+ _base_message_cls = BaseMessage
except ImportError:
- _base_message_cls = None
+ pass
if _base_message_cls is not None and isinstance(value, _base_message_cls):
return message_to_dict(value)
@@ -364,11 +368,15 @@ def json_default(obj: Any) -> Any:
(instead of stringifying their ``repr()``) and falls back to ``str(obj)`` for
anything else, matching the platform's previous ``default=str`` behaviour.
"""
- _base_message_cls: Any
+ # Bind through a separate name: importing *as* the annotated name is a
+ # redefinition, which mypy 2.1 rejects.
+ _base_message_cls: Any = None
try:
- from langchain_core.messages import BaseMessage as _base_message_cls
+ from langchain_core.messages import BaseMessage
+
+ _base_message_cls = BaseMessage
except ImportError:
- _base_message_cls = None
+ pass
if _base_message_cls is not None and isinstance(obj, _base_message_cls):
return message_to_dict(obj)
diff --git a/src/agentomatic/middleware/auth.py b/src/agentomatic/middleware/auth.py
index 8b099b8..6b14f23 100644
--- a/src/agentomatic/middleware/auth.py
+++ b/src/agentomatic/middleware/auth.py
@@ -14,12 +14,13 @@
from starlette.requests import Request
from starlette.responses import JSONResponse, Response
-from agentomatic.middleware.pathutils import path_is_skipped
+from agentomatic.middleware.pathutils import PROBE_PATHS, path_is_skipped
_SKIP_PATHS = {
- "/health",
- "/healthz",
- "/readiness",
+ # Probe endpoints (see PROBE_PATHS) are added below: an orchestrator has
+ # no credentials, so a readiness probe that 401s keeps every pod out of
+ # service and the Deployment never rolls out.
+ *PROBE_PATHS,
"/docs",
"/openapi.json",
"/redoc",
diff --git a/src/agentomatic/middleware/metrics.py b/src/agentomatic/middleware/metrics.py
index f492159..b9ed031 100644
--- a/src/agentomatic/middleware/metrics.py
+++ b/src/agentomatic/middleware/metrics.py
@@ -15,6 +15,8 @@
from starlette.requests import Request
from starlette.responses import Response
+from agentomatic.middleware.pathutils import OPERATIONAL_PATHS
+
try:
from prometheus_client import CONTENT_TYPE_LATEST, Counter, Gauge, Histogram, generate_latest
@@ -27,7 +29,58 @@
generate_latest = cast(Any, None)
CONTENT_TYPE_LATEST = cast(Any, None)
-_SKIP_PATHS = {"/health", "/healthz", "/readiness", "/metrics"}
+#: Probe and scrape traffic is infrastructure noise, not user requests —
+#: recording it would skew request counts and latency histograms.
+_SKIP_PATHS = OPERATIONAL_PATHS
+
+#: Collectors already registered, keyed by metric prefix.
+#:
+#: ``prometheus_client`` registers every collector into one process-wide
+#: registry and raises ``ValueError: Duplicated timeseries`` when the same
+#: metric name is registered twice. Building the collectors in
+#: ``__init__`` therefore made a second ``AgentPlatform.build()`` in one
+#: process a hard crash — which is an ordinary thing to do: uvicorn
+#: ``--reload`` re-imports the app module, embedding hosts serve several
+#: platforms side by side, and test suites build one per case. The metrics
+#: are process-global anyway, so the instances are shared rather than
+#: rebuilt.
+_COLLECTOR_CACHE: dict[str, tuple[Any, Any, Any]] = {}
+
+
+def _collectors(prefix: str) -> tuple[Any, Any, Any]:
+ """Return the ``(requests, duration, active)`` collectors for *prefix*.
+
+ Creates them on first use and reuses them afterwards, so constructing
+ several :class:`MetricsMiddleware` instances in one process is safe.
+
+ Args:
+ prefix: Metric name prefix (e.g. ``agentomatic``).
+
+ Returns:
+ The counter, histogram and gauge registered for *prefix*.
+ """
+ cached = _COLLECTOR_CACHE.get(prefix)
+ if cached is not None:
+ return cached
+ collectors = (
+ Counter(
+ f"{prefix}_http_requests_total",
+ "Total HTTP requests",
+ ["method", "path", "status"],
+ ),
+ Histogram(
+ f"{prefix}_http_request_duration_seconds",
+ "HTTP request duration",
+ ["method", "path"],
+ buckets=(0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0),
+ ),
+ Gauge(
+ f"{prefix}_http_requests_active",
+ "Active HTTP requests",
+ ),
+ )
+ _COLLECTOR_CACHE[prefix] = collectors
+ return collectors
class MetricsMiddleware(BaseHTTPMiddleware):
@@ -39,21 +92,7 @@ def __init__(self, app: Any, *, prefix: str = "agentomatic") -> None:
self._duration: Any | None = None
self._active: Any | None = None
if HAS_PROMETHEUS:
- self._requests = Counter(
- f"{prefix}_http_requests_total",
- "Total HTTP requests",
- ["method", "path", "status"],
- )
- self._duration = Histogram(
- f"{prefix}_http_request_duration_seconds",
- "HTTP request duration",
- ["method", "path"],
- buckets=(0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0),
- )
- self._active = Gauge(
- f"{prefix}_http_requests_active",
- "Active HTTP requests",
- )
+ self._requests, self._duration, self._active = _collectors(prefix)
async def dispatch(
self, request: Request, call_next: Callable[[Request], Awaitable[Response]]
diff --git a/src/agentomatic/middleware/pathutils.py b/src/agentomatic/middleware/pathutils.py
index fa84eee..311b516 100644
--- a/src/agentomatic/middleware/pathutils.py
+++ b/src/agentomatic/middleware/pathutils.py
@@ -2,6 +2,35 @@
from __future__ import annotations
+#: Liveness/readiness probe endpoints — called by orchestrators, not users.
+#:
+#: These must never require credentials and must never be rate limited. A
+#: readiness probe that answers 401 keeps every pod out of service so the
+#: Deployment never rolls out; one that answers 429 under load restarts pods
+#: at exactly the wrong moment. The platform mounts ``/health``, ``/ready``
+#: and ``/readiness``; the ``/healthz``, ``/livez`` and ``/readyz`` spellings
+#: are included so an operator who aliases a conventional path is covered too.
+PROBE_PATHS: frozenset[str] = frozenset(
+ {
+ "/health",
+ "/healthz",
+ "/live",
+ "/livez",
+ "/ready",
+ "/readiness",
+ "/readyz",
+ }
+)
+
+#: Everything infrastructure calls: the probes plus the Prometheus scrape.
+#:
+#: Middleware must not bill or throttle any of these. Behind a NAT, ingress,
+#: or service mesh the kubelet and the scraper share one source IP with real
+#: traffic, so counting them against a per-IP budget makes probes flap to 429
+#: under exactly the load where the pod must stay up — and blanks the metrics
+#: that would explain it.
+OPERATIONAL_PATHS: frozenset[str] = PROBE_PATHS | {"/metrics"}
+
def path_is_skipped(path: str, skip_paths: set[str]) -> bool:
"""Return True when *path* matches an exact skip entry or a prefix entry.
diff --git a/src/agentomatic/middleware/rate_limit.py b/src/agentomatic/middleware/rate_limit.py
index 69a8805..ed2b56f 100644
--- a/src/agentomatic/middleware/rate_limit.py
+++ b/src/agentomatic/middleware/rate_limit.py
@@ -15,7 +15,10 @@
from starlette.requests import Request
from starlette.responses import JSONResponse, Response
-_SKIP_PATHS = {"/health", "/healthz", "/readiness"}
+from agentomatic.middleware.pathutils import OPERATIONAL_PATHS
+
+#: Probe and scrape endpoints are exempt — see ``OPERATIONAL_PATHS``.
+_SKIP_PATHS = OPERATIONAL_PATHS
class RateLimitMiddleware(BaseHTTPMiddleware):
diff --git a/src/agentomatic/optimize/briefing.py b/src/agentomatic/optimize/briefing.py
index e4101b7..477ad90 100644
--- a/src/agentomatic/optimize/briefing.py
+++ b/src/agentomatic/optimize/briefing.py
@@ -129,6 +129,28 @@ def _clip(value: Any, limit: int = 400) -> str:
return text[: limit - 1] + "…"
+def _labelled(label: str, value: Any, limit: int = 400, *, indent: str = "") -> str:
+ """Render ``label: value``, keeping a multi-line value under its label.
+
+ ``AgentExample.to_datapoint`` renders an expected answer as a markdown
+ block ("## Expected answer\n…"). Inlined after a label that produced
+
+ .. code-block:: text
+
+ Expected: ## Expected answer
+ OPT, banana
+
+ — a header mid-line, and the answer itself de-indented out of the item it
+ belongs to. That is the single field a rewrite model most needs to find,
+ so give a multi-line value its own indented block instead.
+ """
+ text = _clip(value, limit)
+ if "\n" not in text:
+ return f"{indent}{label}: {text}"
+ body = "\n".join(f"{indent} {line}" if line.strip() else "" for line in text.splitlines())
+ return f"{indent}{label}:\n{body}"
+
+
def _fmt_json(data: Any, limit: int = 800) -> str:
try:
text = json.dumps(data, ensure_ascii=False, indent=2, default=str)
@@ -204,7 +226,7 @@ def format_dataset_samples(samples: list[Any], *, max_items: int = 8) -> str:
query = getattr(raw, "query", "")
expected = getattr(raw, "expected_answer", None) or getattr(raw, "expected", "")
lines.append(f"{i}. Q: {_clip(query, 220)}")
- lines.append(f" Expected: {_clip(expected, 220)}")
+ lines.append(_labelled("Expected", expected, 220, indent=" "))
return "\n".join(lines)
@@ -231,7 +253,7 @@ def format_eval_io(
score = float(fail.get("score", fail.get("avg_score", 0.0)) or 0.0)
lines.append(f"\n**Failure {idx}** (score={score:.3f})")
lines.append(f"- Input/query: {_clip(fail.get('query'), clip)}")
- lines.append(f"- Expected: {_clip(fail.get('expected'), clip)}")
+ lines.append(_labelled("- Expected", fail.get("expected"), clip))
lines.append(f"- Actual output: {_clip(fail.get('response'), clip)}")
issues = fail.get("feedback") or fail.get("reason") or fail.get("details")
if issues:
@@ -267,7 +289,7 @@ def format_eval_io(
score = float(suc.get("score", suc.get("avg_score", 0.0)) or 0.0)
lines.append(f"\n**Success {idx}** (score={score:.3f})")
lines.append(f"- Input/query: {_clip(suc.get('query'), min(clip, 220))}")
- lines.append(f"- Expected: {_clip(suc.get('expected'), min(clip, 220))}")
+ lines.append(_labelled("- Expected", suc.get("expected"), min(clip, 220)))
lines.append(f"- Actual output: {_clip(suc.get('response'), min(clip, 220))}")
return "\n".join(lines)
diff --git a/src/agentomatic/optimize/fitter.py b/src/agentomatic/optimize/fitter.py
index 911cdf2..c1c8273 100644
--- a/src/agentomatic/optimize/fitter.py
+++ b/src/agentomatic/optimize/fitter.py
@@ -480,6 +480,9 @@ def __init__(
self._resource_registry = ResourceRegistry()
self._trace_store = RolloutTraceStore(path=trace_store_path)
+ #: Set when an evaluation scored nothing at all, so the summary can say
+ #: the reported score is not a measurement.
+ self._eval_blackout: str = ""
self._reward_adapter = MetricRewardAdapter()
# Fitter optimizer — lazy-imported to avoid circular deps
@@ -1415,10 +1418,17 @@ def _record_round(
baseline_config,
best_config,
)
+ blackout = getattr(self, "_eval_blackout", "")
+ if blackout:
+ # Ahead of every other advisory: nothing else in this report means
+ # anything if no datapoint was ever scored.
+ suggestions.insert(0, blackout)
if saturation_warning:
- suggestions.insert(0, saturation_warning)
+ suggestions.insert(1 if blackout else 0, saturation_warning)
if tiny_data_warning:
- suggestions.insert(0 if not saturation_warning else 1, tiny_data_warning)
+ suggestions.insert(
+ sum(1 for flag in (blackout, saturation_warning) if flag), tiny_data_warning
+ )
# Compute metric deltas
metric_deltas: dict[str, float] = {}
@@ -1763,6 +1773,8 @@ async def _evaluate_config(
eval_details: list[dict[str, Any]] = []
scored_count = 0
+ #: Reasons the metric itself refused, for the blackout message below.
+ metric_errors: list[str] = []
for rr in run_results:
messages = trace_adapter.adapt_run_result(rr)
if rr.error:
@@ -1877,6 +1889,7 @@ async def _evaluate_config(
)
except Exception as exc:
logger.warning("Metric evaluation failed for '{}': {}", rr.query[:50], exc)
+ metric_errors.append(f"{type(exc).__name__}: {exc}")
eval_details.append(
{
"query": rr.query,
@@ -1891,7 +1904,27 @@ async def _evaluate_config(
}
)
- # If every point failed evaluation, report 0.0 (not a fabricated mid score).
+ # If every point failed evaluation, report 0.0 (not a fabricated mid
+ # score) -- but say so. A silent 0.0000 is indistinguishable from "the
+ # agent answered everything wrong", and an operator whose server was
+ # simply not running reads a confident verdict on a prompt that was
+ # never exercised.
+ if run_results and not scored_count:
+ # Distinguish "the agent never answered" from "the metric could not
+ # score the answers": they point at completely different fixes.
+ call_errors = [str(rr.error) for rr in run_results if rr.error]
+ if len(call_errors) == len(run_results):
+ cause = f"all {len(run_results)} agent call(s) failed"
+ first_error = call_errors[0]
+ else:
+ cause = f"the metric scored none of {len(run_results)} response(s)"
+ first_error = metric_errors[0] if metric_errors else ""
+ self._eval_blackout = (
+ f"No datapoint could be evaluated: {cause}. The score below is "
+ "not a measurement."
+ + (f" First error: {first_error[:200]}" if first_error else "")
+ )
+ logger.error("❌ {}", self._eval_blackout)
avg_score = sum(scores) / scored_count if scored_count else 0.0
per_dim: dict[str, float] = {
dim: sum(vals) / len(vals) for dim, vals in dim_accumulators.items() if vals
diff --git a/src/agentomatic/optimize/metrics.py b/src/agentomatic/optimize/metrics.py
index b8e30c0..81449f2 100644
--- a/src/agentomatic/optimize/metrics.py
+++ b/src/agentomatic/optimize/metrics.py
@@ -19,6 +19,7 @@
import difflib
import os
+import re
from abc import ABC, abstractmethod
from collections.abc import Callable
from dataclasses import dataclass, field
@@ -106,6 +107,45 @@ async def evaluate(
# =====================================================================
+#: Header that opens the plain answer inside a rich expected reference.
+_EXPECTED_ANSWER_HEADER = re.compile(
+ r"^[ \t]*##[ \t]*Expected answer[ \t]*$", re.MULTILINE | re.IGNORECASE
+)
+#: Any other section header in such a reference, which ends the answer.
+_ANY_SECTION_HEADER = re.compile(r"^[ \t]*##[ \t]*\S", re.MULTILINE)
+
+
+def plain_expected(expected: str | None) -> str | None:
+ """Return the literal ground-truth answer inside an expected reference.
+
+ ``AgentExample.to_datapoint`` builds a *judge-facing* reference — judge
+ guidance, a rubric, an "## Expected answer" section, the structured
+ output as JSON. An LLM judge reads all of that. A deterministic metric
+ cannot: comparing a response against markdown headers scores near zero no
+ matter how right the answer is, so ``fit()`` over an ``AgentDataset``
+ reported "no improvement" forever, whatever the optimizer proposed.
+
+ Plain strings pass through untouched, so a hand-written dataset behaves
+ exactly as before.
+
+ Args:
+ expected: The expected value as the dataset carries it.
+
+ Returns:
+ Just the answer text, or ``expected`` when there is no such section.
+ """
+ if not expected:
+ return expected
+ opener = _EXPECTED_ANSWER_HEADER.search(expected)
+ if opener is None:
+ return expected
+ rest = expected[opener.end() :].lstrip("\n")
+ nxt = _ANY_SECTION_HEADER.search(rest)
+ body = rest[: nxt.start()] if nxt else rest
+ stripped = "\n".join(line.strip() for line in body.splitlines()).strip()
+ return stripped or expected
+
+
class ExactMatchMetric(BaseMetric):
"""Simple string matching — no LLM required."""
@@ -126,6 +166,7 @@ async def evaluate(
return EvalResult(
metric_name=self.name, score=0.0, reason="No expected answer provided"
)
+ expected = plain_expected(expected) or expected
if self.fuzzy:
ratio = difflib.SequenceMatcher(
@@ -160,8 +201,9 @@ async def evaluate(
if expected is None:
return EvalResult(metric_name=self.name, score=0.0, reason="No expected answer")
+ expected = plain_expected(expected) or expected
resp_lower = response.lower()
- keywords = [kw.strip() for kw in expected.lower().split(",")]
+ keywords = [kw.strip() for kw in expected.lower().split(",") if kw.strip()]
found = sum(1 for kw in keywords if kw in resp_lower)
score = found / len(keywords) if keywords else 0.0
return EvalResult(
diff --git a/src/agentomatic/pipelines/context.py b/src/agentomatic/pipelines/context.py
index d4a8ba8..3199fe2 100644
--- a/src/agentomatic/pipelines/context.py
+++ b/src/agentomatic/pipelines/context.py
@@ -207,7 +207,9 @@ def resolve_mapping(self, mapping: dict[str, Any]) -> dict[str, Any]:
def to_eval_namespace(self) -> dict[str, Any]:
"""Create a namespace dict for evaluating condition expressions.
- The namespace exposes ``ctx`` (this context) and ``len``.
+ The namespace exposes ``ctx`` (this context) plus a small set of
+ safe builtins; ``__builtins__`` itself is stripped by the caller, so
+ anything absent here is unavailable to a condition.
"""
return {
"ctx": self,
diff --git a/src/agentomatic/pipelines/engine.py b/src/agentomatic/pipelines/engine.py
index 823a9fd..68f78dc 100644
--- a/src/agentomatic/pipelines/engine.py
+++ b/src/agentomatic/pipelines/engine.py
@@ -43,6 +43,22 @@
)
from .validation import validate_against_schema
+#: How deep ``sub_pipeline`` steps may nest before the engine refuses.
+#:
+#: Validation catches reference cycles statically, but pipelines are editable
+#: over HTTP: a sub-pipeline can be saved after an engine was built, so this
+#: bounds the damage even when the static check could not have seen it.
+MAX_SUB_PIPELINE_DEPTH = 10
+
+
+class ConditionError(RuntimeError):
+ """A step condition could not be evaluated.
+
+ Distinct from a condition that evaluated falsy: that is a routing
+ decision, this is a broken pipeline.
+ """
+
+
if TYPE_CHECKING:
from collections.abc import Awaitable, Callable
@@ -65,6 +81,8 @@ class PipelineEngine:
registry: The agent registry for resolving agent names.
sub_pipelines: Optional dict of named sub-pipelines for
``sub_pipeline`` steps.
+ depth: Sub-pipeline nesting depth. Set by the engine when it
+ recurses; callers should leave it at ``0``.
Example::
@@ -82,10 +100,13 @@ def __init__(
endpoints: EndpointRegistry | None = None,
ingestors: IngestionRegistry | None = None,
plugins: PluginRegistry | None = None,
+ depth: int = 0,
) -> None:
self.config = config
self.registry = registry
self.sub_pipelines = sub_pipelines or {}
+ #: How many sub-pipeline frames deep this engine is running.
+ self.depth = depth
self.endpoints = endpoints
self.ingestors = ingestors
self.plugins = plugins
@@ -131,11 +152,27 @@ def validate(self) -> list[str]:
f"Available: {self.registry.list_names()}"
)
- # Check sub-pipeline references
+ # Check sub-pipeline references, and that following them terminates.
for step in self.config.steps:
if isinstance(step, SubPipelineStepConfig):
if step.pipeline not in self.sub_pipelines:
errors.append(f"Sub-pipeline '{step.pipeline}' not found")
+ errors.extend(self._sub_pipeline_cycles())
+
+ # Conditions are Python expressions; a syntax error in one is worth
+ # reporting before the pipeline runs rather than at the step.
+ for step in self.config.steps:
+ condition = getattr(step, "condition", None)
+ if not condition:
+ continue
+ try:
+ compile(condition, f"", "eval")
+ except SyntaxError as exc:
+ errors.append(
+ f"Step '{step.name}' has an invalid condition {condition!r}: "
+ f"{exc.msg}. Conditions are Python expressions over `ctx`, "
+ "not `$.` mappings."
+ )
# Check plugin references
required_plugins = self.config.get_plugin_names()
@@ -352,16 +389,39 @@ async def _execute_steps(
# Evaluate condition if present
condition = getattr(step_config, "condition", None)
- if condition and not self._evaluate_condition(condition, ctx):
- logger.info(f" ⏭️ Skipping '{step_config.name}' (condition not met)")
- pipeline_result.steps[step_config.name] = StepResult(
- name=step_config.name,
- status=StepStatus.SKIPPED,
- )
- await self._report_step_progress(
- exec_pos + 1, total_steps, step_config.name, "skipped"
- )
- continue
+ if condition:
+ try:
+ should_run = self._evaluate_condition(condition, ctx)
+ except ConditionError as exc:
+ # Surface it as a failed step so the pipeline's on_error
+ # policy decides — never as a silent skip under a
+ # successful pipeline.
+ logger.error(f" ❌ Step '{step_config.name}': {exc}")
+ result = StepResult(
+ name=step_config.name,
+ status=StepStatus.FAILED,
+ error=str(exc),
+ )
+ pipeline_result.steps[step_config.name] = result
+ unsuccessful_steps.add(step_config.name)
+ await self._report_step_progress(
+ exec_pos + 1, total_steps, step_config.name, "failed"
+ )
+ if step_config.on_error is ErrorPolicy.SKIP:
+ continue
+ pipeline_result.status = PipelineStatus.FAILED
+ pipeline_result.error = f"Step '{step_config.name}' failed: {exc}"
+ break
+ if not should_run:
+ logger.info(f" ⏭️ Skipping '{step_config.name}' (condition not met)")
+ pipeline_result.steps[step_config.name] = StepResult(
+ name=step_config.name,
+ status=StepStatus.SKIPPED,
+ )
+ await self._report_step_progress(
+ exec_pos + 1, total_steps, step_config.name, "skipped"
+ )
+ continue
logger.info(f" ▶️ Executing step '{step_config.name}'")
await self._report_step_progress(exec_pos, total_steps, step_config.name, "running")
@@ -584,6 +644,37 @@ async def _report_progress(
except Exception as exc: # noqa: BLE001
logger.debug(f"pipeline progress_cb failed: {exc}")
+ def _sub_pipeline_cycles(self) -> list[str]:
+ """Report sub-pipeline reference cycles reachable from this pipeline.
+
+ Every discovered pipeline is available as a sub-pipeline, so a
+ pipeline can reference itself — directly, or around a longer loop.
+ Executing one would recurse until the worker died, and pipelines are
+ editable over HTTP, so this has to be caught before it runs rather
+ than only bounded at runtime.
+
+ Returns:
+ One message per cycle found, naming the path.
+ """
+ cycles: list[str] = []
+ start = self.config.name
+
+ def walk(name: str, path: list[str], seen: set[str]) -> None:
+ config = self.sub_pipelines.get(name) if name != start else self.config
+ if config is None:
+ return
+ for step in config.steps:
+ if not isinstance(step, SubPipelineStepConfig):
+ continue
+ nxt = step.pipeline
+ if nxt in seen:
+ cycles.append("Sub-pipeline cycle: " + " -> ".join([*path, nxt]))
+ continue
+ walk(nxt, [*path, nxt], seen | {nxt})
+
+ walk(start, [start], {start})
+ return cycles
+
async def _execute_sub_pipeline(
self,
config: SubPipelineStepConfig,
@@ -592,6 +683,18 @@ async def _execute_sub_pipeline(
"""Execute a nested sub-pipeline."""
t0 = time.perf_counter()
+ if self.depth >= MAX_SUB_PIPELINE_DEPTH:
+ # Backstop for a cycle that validation did not see — for instance
+ # a sub-pipeline saved after this engine was built.
+ return StepResult(
+ name=config.name,
+ status=StepStatus.FAILED,
+ error=(
+ f"Sub-pipeline nesting exceeded {MAX_SUB_PIPELINE_DEPTH} levels at "
+ f"'{config.pipeline}' — check for a reference cycle"
+ ),
+ )
+
sub_config = self.sub_pipelines.get(config.pipeline)
if sub_config is None:
return StepResult(
@@ -615,6 +718,7 @@ async def _execute_sub_pipeline(
endpoints=self.endpoints,
ingestors=self.ingestors,
plugins=self.plugins,
+ depth=self.depth + 1,
)
sub_result = await asyncio.wait_for(
sub_engine.run(sub_input),
@@ -672,13 +776,34 @@ async def _run_rollbacks(
pipeline_result.metadata["rolled_back_steps"] = compensated
def _evaluate_condition(self, condition: str, ctx: PipelineContext) -> bool:
- """Safely evaluate a condition expression."""
+ """Evaluate a step condition.
+
+ Args:
+ condition: A Python expression over ``ctx`` (see the pipelines
+ guide) deciding whether the step runs.
+ ctx: The live pipeline context.
+
+ Returns:
+ Whether the step should run.
+
+ Raises:
+ ConditionError: If the expression cannot be evaluated at all —
+ a typo, a renamed step, or ``$.`` mapping syntax used where a
+ ``ctx`` expression belongs. That is a defect in the pipeline,
+ not a decision to skip, and it used to be swallowed into
+ ``False``: the branch silently never fired and the pipeline
+ still reported ``success``.
+ """
+ ns = ctx.to_eval_namespace()
try:
- ns = ctx.to_eval_namespace()
return bool(eval(condition, {"__builtins__": {}}, ns)) # noqa: S307
except Exception as exc:
- logger.warning(f"Condition eval failed: {exc}")
- return False
+ raise ConditionError(
+ f"condition {condition!r} could not be evaluated "
+ f"({type(exc).__name__}: {exc}). Conditions are Python "
+ "expressions over `ctx` — e.g. "
+ "ctx.get_step_output('step').get('field') — not `$.` mappings."
+ ) from exc
def _apply_output_mapping(
self,
diff --git a/src/agentomatic/pipelines/loader.py b/src/agentomatic/pipelines/loader.py
index 950ae6b..4c67a73 100644
--- a/src/agentomatic/pipelines/loader.py
+++ b/src/agentomatic/pipelines/loader.py
@@ -148,7 +148,21 @@ def _parse_agent_step(data: dict[str, Any]) -> AgentStepConfig:
Returns:
A validated ``AgentStepConfig``.
+
+ Raises:
+ ValueError: If the step declares no ``agent``. The message names the
+ step, because a bare ``KeyError('agent')`` reached the operator as
+ ``failed to load: 'agent'`` — which says neither which step nor
+ what to add, and the pipeline is then simply absent at runtime.
"""
+ if "agent" not in data:
+ label = data.get("name") or ""
+ declared = ", ".join(sorted(data)) or "nothing"
+ raise ValueError(
+ f"Step '{label}' has no 'agent' key (it declares: {declared}). "
+ "Steps nested under 'parallel' must be agent steps; plugin, "
+ "endpoint and ingestion steps have to sit at the top level."
+ )
agent: str = data["agent"]
name: str = data.get("name", agent)
@@ -708,7 +722,12 @@ def discover_pipelines(
try:
cfg = PipelineLoader.from_yaml(path)
except Exception as exc: # noqa: BLE001
- logger.warning("Skipping {} – failed to load: {}", path, exc)
+ logger.warning(
+ "Skipping pipeline {} – it will NOT be served: {}: {}",
+ path,
+ type(exc).__name__,
+ exc,
+ )
continue
if cfg.name in configs:
@@ -747,7 +766,12 @@ def discover_pipeline_files(directory: Path) -> dict[str, Path]:
try:
cfg = PipelineLoader.from_yaml(path)
except Exception as exc: # noqa: BLE001
- logger.warning("Skipping {} – failed to load: {}", path, exc)
+ logger.warning(
+ "Skipping pipeline {} – it will NOT be served: {}: {}",
+ path,
+ type(exc).__name__,
+ exc,
+ )
continue
if cfg.name in files:
continue # first occurrence wins, matching discover_pipelines
diff --git a/src/agentomatic/pipelines/router.py b/src/agentomatic/pipelines/router.py
index 17ced85..1e66d57 100644
--- a/src/agentomatic/pipelines/router.py
+++ b/src/agentomatic/pipelines/router.py
@@ -175,10 +175,15 @@ def _engine_for(config: PipelineConfig) -> PipelineEngine:
"""Build an engine for an arbitrary (possibly unsaved) config."""
from .engine import PipelineEngine
+ # Every served pipeline doubles as a sub-pipeline, so `sub_pipeline`
+ # steps can compose what the platform already discovered. Read
+ # `all_pipelines` here rather than at router-build time so a pipeline
+ # saved through the builder is immediately referencable. Explicit
+ # `sub_pipelines` win on a name clash.
return PipelineEngine(
config,
registry,
- all_sub,
+ {**all_pipelines, **all_sub},
endpoints=endpoints,
ingestors=ingestors,
plugins=plugins,
diff --git a/src/agentomatic/providers/llm.py b/src/agentomatic/providers/llm.py
index 1458025..16e88dc 100644
--- a/src/agentomatic/providers/llm.py
+++ b/src/agentomatic/providers/llm.py
@@ -148,6 +148,27 @@ async def my_llm(messages):
logger.info(f"Global LLM set to: {type(instance).__name__}")
+#: The pip extra that supplies each provider's client library.
+_PROVIDER_EXTRAS: dict[str, str] = {
+ "ollama": "ollama",
+ "openai": "openai",
+ "openai_compatible": "openai",
+ "azure": "azure",
+ "vertex": "vertex",
+}
+
+
+class LLMDriverMissingError(RuntimeError):
+ """A configured LLM provider's client library is not installed.
+
+ Distinct from a backend that is merely unreachable. An unreachable
+ backend may recover; a missing driver never will, and until this was
+ raised the platform quietly substituted a dummy model — so a deployment
+ that had configured a real provider booted healthy and answered every
+ request with fabricated text.
+ """
+
+
def _build_llm(provider: str, **kwargs: Any) -> Any:
"""Build an LLM instance for the given provider.
@@ -574,8 +595,28 @@ def get_named_llm(
)
_named_instances[name] = built
logger.debug(f"Created named LLM instance '{name}' ({provider})")
+ except ImportError as exc:
+ extra = _PROVIDER_EXTRAS.get(provider.lower())
+ install = (
+ f"pip install 'agentomatic[{extra}]'"
+ if extra
+ else "install the provider's client library"
+ )
+ raise LLMDriverMissingError(
+ f"LLM '{name}' is configured for provider '{provider}', but its "
+ f"client library is not installed ({exc}). Fix the image rather "
+ f"than the request: {install}. Refusing to substitute a dummy "
+ "model — that would answer every call with fabricated text while "
+ "the platform reported healthy."
+ ) from exc
except Exception as exc:
- logger.warning(f"Failed to build LLM '{name}' ({provider}): {exc}. Using dummy.")
+ # The backend may simply be down; that can recover, so keep the
+ # dummy so local development still runs. It is logged loudly
+ # because every answer from here on is fake.
+ logger.warning(
+ f"Failed to build LLM '{name}' ({provider}): {exc}. "
+ "Using a DUMMY model — responses are fabricated."
+ )
built = _build_dummy_llm()
_named_instances[name] = built
return built
diff --git a/src/agentomatic/security/jwt_auth.py b/src/agentomatic/security/jwt_auth.py
index 9fde8c6..f082d2d 100644
--- a/src/agentomatic/security/jwt_auth.py
+++ b/src/agentomatic/security/jwt_auth.py
@@ -22,7 +22,7 @@
from starlette.requests import Request
from starlette.responses import JSONResponse, Response
-from agentomatic.middleware.pathutils import path_is_skipped
+from agentomatic.middleware.pathutils import PROBE_PATHS, path_is_skipped
from agentomatic.security.claims import extract_roles, extract_scopes
from agentomatic.security.dpop import DPoPConfig, DPoPError, validate_dpop
@@ -46,9 +46,9 @@
# ---------------------------------------------------------------------------
_DEFAULT_SKIP_PATHS: set[str] = {
- "/health",
- "/healthz",
- "/readiness",
+ # Probe endpoints (see PROBE_PATHS): an orchestrator carries no bearer
+ # token, so a readiness probe that 401s keeps every pod out of service.
+ *PROBE_PATHS,
"/docs",
"/openapi.json",
"/redoc",
diff --git a/src/agentomatic/storage/models.py b/src/agentomatic/storage/models.py
index 81bacc0..4fede71 100644
--- a/src/agentomatic/storage/models.py
+++ b/src/agentomatic/storage/models.py
@@ -18,6 +18,29 @@
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
+def iso_utc(value: datetime | None) -> str | None:
+ """Render a stored timestamp as an unambiguous UTC ISO-8601 string.
+
+ Every timestamp in this schema is written as UTC (``datetime.now(UTC)``),
+ but not every backend hands it back that way: ``DateTime(timezone=True)``
+ is a no-op on SQLite, so a value read from there comes back naive while
+ the same value from Postgres carries ``+00:00``. Emitting the raw
+ ``isoformat()`` therefore made the API's timestamp format depend on which
+ database was configured, leaving clients to guess a naive string's zone.
+
+ Args:
+ value: A stored datetime, naive (assumed UTC) or aware.
+
+ Returns:
+ An ISO-8601 string with an explicit offset, or ``None``.
+ """
+ if value is None:
+ return None
+ if value.tzinfo is None:
+ value = value.replace(tzinfo=UTC)
+ return value.astimezone(UTC).isoformat()
+
+
class Base(DeclarativeBase):
"""SQLAlchemy declarative base."""
@@ -64,8 +87,8 @@ def to_dict(self) -> dict[str, Any]:
"agent_name": self.agent_name,
"title": self.title,
"metadata": self.metadata_json or {},
- "created_at": self.created_at.isoformat() if self.created_at else None,
- "updated_at": self.updated_at.isoformat() if self.updated_at else None,
+ "created_at": iso_utc(self.created_at),
+ "updated_at": iso_utc(self.updated_at),
"message_count": self.message_count,
"parent_thread_id": self.parent_thread_id,
"fork_message_index": self.fork_message_index,
@@ -97,7 +120,7 @@ def to_dict(self) -> dict[str, Any]:
"role": self.role,
"content": self.content,
"metadata": self.metadata_json or {},
- "timestamp": self.created_at.isoformat() if self.created_at else None,
+ "timestamp": iso_utc(self.created_at),
}
@@ -132,7 +155,7 @@ def to_dict(self) -> dict[str, Any]:
"rating": self.rating,
"comment": self.comment,
"feedback_type": self.feedback_type,
- "created_at": self.created_at.isoformat() if self.created_at else None,
+ "created_at": iso_utc(self.created_at),
}
@@ -165,8 +188,8 @@ def to_dict(self) -> dict[str, Any]:
"agent_name": self.agent_name,
"node_name": self.node_name,
"state_snapshot": self.state_json,
- "created_at": self.created_at.isoformat() if self.created_at else None,
- "expires_at": self.expires_at.isoformat() if self.expires_at else None,
+ "created_at": iso_utc(self.created_at),
+ "expires_at": iso_utc(self.expires_at),
}
@@ -200,7 +223,7 @@ def to_dict(self) -> dict[str, Any]:
"parent_checkpoint_id": self.parent_checkpoint_id,
"checkpoint": self.checkpoint_json,
"metadata": self.metadata_json,
- "created_at": self.created_at.isoformat() if self.created_at else None,
+ "created_at": iso_utc(self.created_at),
}
@@ -247,7 +270,7 @@ def to_dict(self) -> dict[str, Any]:
"agent_name": self.agent_name, # BC alias
"thread_id": self.thread_id,
"run_id": self.run_id,
- "timestamp": self.timestamp.isoformat() if self.timestamp else None,
+ "timestamp": iso_utc(self.timestamp),
"endpoint": self.endpoint,
"input": self.input_json or {},
"output": self.output_json or {},
@@ -297,7 +320,7 @@ def to_dict(self) -> dict[str, Any]:
"status": self.status,
"recommendations": self.recommendations or [],
"metadata": self.metadata_json or {},
- "created_at": self.created_at.isoformat() if self.created_at else None,
+ "created_at": iso_utc(self.created_at),
}
@@ -338,5 +361,5 @@ def to_dict(self) -> dict[str, Any]:
"learnings": self.learnings or [],
"artefacts": self.artefacts or {},
"metadata": self.metadata_json or {},
- "created_at": self.created_at.isoformat() if self.created_at else None,
+ "created_at": iso_utc(self.created_at),
}
diff --git a/src/agentomatic/storage/sqlalchemy.py b/src/agentomatic/storage/sqlalchemy.py
index 777ece3..9f3daeb 100644
--- a/src/agentomatic/storage/sqlalchemy.py
+++ b/src/agentomatic/storage/sqlalchemy.py
@@ -140,6 +140,14 @@ def _enable_sqlite_fk(dbapi_connection: Any, _connection_record: Any) -> None:
cursor.execute("PRAGMA foreign_keys=ON")
cursor.close()
+ # ``expire_on_commit=False`` keeps every attribute loaded after a
+ # commit. Combined with the Python-side column defaults in
+ # ``storage.models`` (there are no server defaults), a committed row is
+ # already complete — so the write paths below deliberately do *not*
+ # call ``session.refresh()``. Doing so issued a second round trip per
+ # write purely to re-read values the object already held, which on a
+ # networked Postgres roughly doubled the latency of every logged
+ # invocation.
self._session_factory = async_sessionmaker(
self._engine,
expire_on_commit=False,
@@ -221,7 +229,6 @@ async def create_thread(
)
session.add(thread)
await session.commit()
- await session.refresh(thread)
return thread.to_dict()
async def get_thread(self, thread_id: str) -> dict[str, Any] | None:
@@ -283,7 +290,6 @@ async def update_thread(
setattr(thread, key, val)
thread.updated_at = datetime.now(UTC)
await session.commit()
- await session.refresh(thread)
return thread.to_dict()
# ------------------------------------------------------------------
@@ -314,7 +320,6 @@ async def add_message(
thread.message_count += 1
thread.updated_at = datetime.now(UTC)
await session.commit()
- await session.refresh(msg)
return msg.to_dict()
async def get_messages(
@@ -364,7 +369,6 @@ async def add_feedback(
)
session.add(fb)
await session.commit()
- await session.refresh(fb)
return fb.to_dict()
async def get_feedback(
@@ -425,7 +429,6 @@ async def save_suspended_state(
)
session.add(suspended)
await session.commit()
- await session.refresh(suspended)
return suspended.to_dict()
async def get_suspended_state(self, approval_id: str) -> dict[str, Any] | None:
@@ -546,7 +549,6 @@ async def fork_thread(
forked_thread.message_count = forked_count
await session.commit()
- await session.refresh(forked_thread)
return forked_thread.to_dict()
async def get_thread_lineage(self, thread_id: str) -> dict[str, Any]:
@@ -730,7 +732,6 @@ async def create_invocation_log(
row = AgentInvocationLogModel(**kwargs)
session.add(row)
await session.commit()
- await session.refresh(row)
return row.to_dict()
async def get_invocation_log(self, log_id: str) -> dict[str, Any] | None:
@@ -844,7 +845,6 @@ async def save_log_analysis(
row = LogAnalysisModel(**kwargs)
session.add(row)
await session.commit()
- await session.refresh(row)
return row.to_dict()
async def get_latest_log_analysis(
@@ -921,7 +921,6 @@ async def create_optimization_run(
row = OptimizationRunModel(**kwargs)
session.add(row)
await session.commit()
- await session.refresh(row)
return row.to_dict()
async def get_optimization_run(self, run_id: str) -> dict[str, Any] | None:
diff --git a/src/agentomatic/studio/static/LOCAL_PATCHES.md b/src/agentomatic/studio/static/LOCAL_PATCHES.md
new file mode 100644
index 0000000..0321cb8
--- /dev/null
+++ b/src/agentomatic/studio/static/LOCAL_PATCHES.md
@@ -0,0 +1,81 @@
+# Local patches to the built Studio bundle
+
+The Studio UI is built in a separate repository; this directory holds only its
+compiled output. Three fixes below were applied **directly to the built assets**
+because the defects are user-visible in every deployment. Each needs the
+equivalent change upstream, after which these notes can go.
+
+Until then, note that a frontend rebuild silently reverts all three.
+
+---
+
+## 1. `connectionStatus` never reached `"connected"`
+
+**File:** `static/js/main.8ca7b978.js`
+
+`ConnectionSetup.handleConnect` calls `setIsConnected(true)` but never
+`setConnectionStatus('connected')`. The header renders `connectionStatus`,
+which starts at `'disconnected'`, so after a *successful* connect the UI
+showed "Connected" in one indicator and "Disconnected" — with a Retry
+button — in another, while every backend call was returning 200.
+`connectionStatus` only became `'connected'` via `attemptReconnection()`,
+i.e. only after a failure and recovery.
+
+The patch makes the store setter keep both flags in agreement, since they
+must never disagree:
+
+```js
+// before
+setIsConnected:t=>e({isConnected:t})
+// after
+setIsConnected:t=>e({isConnected:t,connectionStatus:t?"connected":"disconnected"})
+```
+
+**Upstream fix** — in `store/useStudioStore.ts`:
+
+```ts
+setIsConnected: (isConnected) =>
+ set({ isConnected, connectionStatus: isConnected ? 'connected' : 'disconnected' }),
+```
+
+## 2. Google Fonts were fetched at page load
+
+**File:** `static/css/main.961204dc.css`
+
+The stylesheet opened with two `@import url(https://fonts.googleapis.com/...)`
+rules for Inter and JetBrains Mono. A self-hosted admin UI should not fetch
+assets from a third party at page load: it breaks in air-gapped or
+egress-restricted deployments (where the request hangs or resets before the
+page paints) and it sends every viewer's IP and User-Agent to Google, which
+is a compliance question for enterprise operators.
+
+Both `@import` lines were removed. Every `font-family` in the bundle already
+declared a full fallback stack (system UI font, then a standard monospace),
+so the UI renders natively with no external request.
+
+**Upstream fix**: drop the imports and either accept the system stack or
+self-host the WOFF2 files with a local `@font-face`.
+
+## 3. No favicon was declared
+
+**File:** `index.html`
+
+Nothing declared an icon, so every browser fell back to `/favicon.ico` at the
+origin root — a path the platform does not serve. That produced a 404 in
+every deployment's access log and a console error for every Studio user.
+
+An inline `data:image/svg+xml` icon was added before ``, which costs no
+request at all. `imgs/logo.png` was not used: it is a 588 KB JPEG (despite the
+extension) at 1024x1024, far too heavy for a tab icon.
+
+**Upstream fix**: declare an icon in `public/index.html`, inline or as a small
+self-hosted `.svg`/`.ico`.
+
+---
+
+## Source maps
+
+`main.8ca7b978.js.map` was **not** regenerated after patch 1. The added
+characters shift every column mapping that follows on that line, so
+positions in the map are approximate from that point on. Regenerate the
+bundle upstream rather than trusting the map around the store definition.
diff --git a/src/agentomatic/studio/static/index.html b/src/agentomatic/studio/static/index.html
index 36e6092..8c0f981 100644
--- a/src/agentomatic/studio/static/index.html
+++ b/src/agentomatic/studio/static/index.html
@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#7c3aed"/><meta name="description" content="Agentomatic Studio - Visual interface for debugging and testing Agentomatic agents"/><link rel="manifest" href="/studio/ui/manifest.json"/><title>Agentomatic Studio