From bda82eef98a2f0ba88279de0c968f2634b3ae71c Mon Sep 17 00:00:00 2001 From: bwerapol Date: Tue, 28 Jul 2026 21:04:28 +0000 Subject: [PATCH 1/3] docs: scope-filter blindness, corpus capture gap, and the Redis config drift MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three findings from building the second generation-uplift suite. That suite takes its treatment-arm injection from the LIVE KRA path rather than a hand-written block; the first probe returned a null, and chasing it surfaced more than the benchmark would have. 1. Capture gap (partially closed). Two of three probed repo rules were absent from the Brain entirely — force-dynamic and the package boundary — despite each having a class-of-bug entry and real debugging cost. Proven to be capture, not retrieval: teaching force-dynamic and re-running the identical prompt ranked it FIRST. Nine rules backfilled; the rest of GUIDELINES is unswept. 2. scope-filter blindness (open). buildKnowledgeWhere and buildRawProjectFilter resolve visibility purely on ownerProjectId and never consult the scope column, so scope:"user" rows carrying a non-null ownerProjectId are invisible outside their capture project — 117 rows. Measured: the best-matching item in the corpus (0.9009 similarity, 11 successful uses, exact trigger match) is returned first by unscoped retrieval and not at all by the project-scoped session path. ~Half the corpus (101 in Default vs 100 in Brain Platform) is invisible to a BrainPlatform session. Left open deliberately: the fix changes what every user sees. 3. Redis drift (doc fixed). .env.example claimed "Production docker-compose provides redis://redis:6379". It does not — compose only passes ${REDIS_URL:-} through. So a healthy redis container has run for weeks with nothing connected, rate-limit state lives in an in-process Map that resets on every deploy, and the get-then-set race fixed in #173 was a live production bypass rather than a dev-only concern. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01Rqj5y9fT3XPUqnqKyustp9 --- .env.example | 12 ++++++++++-- docs/KNOWN_ISSUES.md | 21 +++++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 891ea8f..488c64f 100644 --- a/.env.example +++ b/.env.example @@ -248,8 +248,16 @@ SENTRY_DSN="" SENTRY_TRACES_SAMPLE_RATE="0" # 0..1 — leave at 0 until you've sized the volume # --- Redis (Wave 2 — multi-replica rate-limit state) ------------------------- -# Dev / single-host deploys can leave empty; the proxy falls back to an -# in-process Map. Production docker-compose provides `redis://redis:6379`. +# You MUST set this yourself for a server deploy — compose does NOT provide it. +# deploy/docker-compose.yml only passes the value through (`REDIS_URL: +# ${REDIS_URL:-}`), so leaving it empty here means the `redis` service runs but +# nothing connects to it, and rate-limit state lives in an in-process Map. +# +# In-process state is correct on a single replica but resets on every deploy or +# `reload.sh web` — which silently clears daily caps such as +# RATE_LIMIT_MEETING_EXTRACT_PER_DAY. Set the line below on any server deploy: +# REDIS_URL="redis://redis:6379" +# Local dev can leave it empty. REDIS_URL="" # --- Email (optional — falls back to manual-link UX when disabled) ---------- diff --git a/docs/KNOWN_ISSUES.md b/docs/KNOWN_ISSUES.md index b3a4ee1..2d65cd2 100644 --- a/docs/KNOWN_ISSUES.md +++ b/docs/KNOWN_ISSUES.md @@ -384,6 +384,27 @@ compose allowlist for the `qwen3-coder` default, or override --- +## 0p. Corpus capture gap + scope-filter blindness (2026-07-28) + +Found while building the second generation-uplift suite (#126 follow-up). That +suite's treatment arm takes its injected block from the **live KRA path** rather +than a hand-written file — the change you make when you want to measure the +product instead of the mechanism. The first probe returned a null, and chasing +it produced two defects and one measurement. + +**Method (reproducible).** Ask a well-formed technical question whose answer is a +documented repo rule, via both `brain_start_session(projectName: +"BrainPlatform")` (the production path) and `brain_retrieve_knowledge` +(unscoped), then compare what comes back. + +| Issue | Where | Status | +|---|---|---| +| ~~**Hard-won repo rules were never taught to the Brain.**~~ **Partially closed (2026-07-28).** A probe of three documented rules found two absent from the corpus entirely: the `force-dynamic` rule (§10) and the package-boundary rule — each with a class-of-bug entry here and real debugging cost behind it. The Oracle said so outright: *"isn't captured in your current knowledge base."* Only the `#418` mount-gate rule was present. **This is a capture gap, not a retrieval gap** — proven by teaching `force-dynamic` and re-running the *identical* prompt, which then ranked it **first**. KRA ranked it correctly the moment it existed. Nine rules were backfilled from GUIDELINES §9/§10 and AGENTS.md. **The rest of GUIDELINES/KNOWN_ISSUES has not been swept** — the docs remain the de facto memory for pre-Stage-1 lessons. | `docs/GUIDELINES.md`, the Brain corpus | partially done | +| **`scope: "user"` knowledge is invisible outside the project it was captured in — 117 rows affected.** Both `buildKnowledgeWhere` and `buildRawProjectFilter` (the pgvector path that `kra.ts` and `oracle.ts` actually use) resolve visibility as `ownerProjectId = $activeProject OR (ownerProjectId IS NULL AND ownerUserId = $user)`. **The `scope` column is never consulted.** A row with `scope: "user"` and a non-null `ownerProjectId` therefore matches neither arm and is filtered out — even though `scope: "user"` is `brain_teach_knowledge`'s *default* and reads as "follows the user". (`Knowledge.visibility` is a separate Phase-4 field and does not govern this path.) **Measured impact, same prompt both arms:** unscoped retrieval ranked `cmqpqemoh…` — trigger *"Adding a user-facing concept or glossary page to the External Brain webapp"*, 100% success over 11 uses — **first, at 0.9009 similarity**. The project-scoped session path did not return it at all, injecting a tangential i18n rule instead. **Scale:** 101 active items in `Default` vs 100 in `Brain Platform`; 117 rows repo-wide are `scope='user'` with a non-null project. Roughly half the corpus is invisible to a `BrainPlatform` session. **Not fixed here** — the fix is a design call with real blast radius (honour `scope` in the filter, vs. null out `ownerProjectId` on user-scope rows as a data repair) and it changes what every user sees. Note the duplicate-project detector cannot surface this: it looks for *normalized name collisions*, and `Brain Platform` vs `Default` will never collide. | `packages/core/src/scope-filter.ts:64-75,155-168`; `kra.ts:172` | open — needs a design call | +| ~~**`.env.example` claimed compose provides Redis; it does not.**~~ **Doc fixed (2026-07-28).** The comment read *"Production docker-compose provides `redis://redis:6379`"*, but `deploy/docker-compose.yml` only passes the value through (`REDIS_URL: ${REDIS_URL:-}`). Consequence on the reference instance: a healthy `deploy-redis-1` container has been up for weeks with **nothing connected to it**, and rate-limit state lives in an in-process Map. Correct on a single replica, but it resets on every deploy / `reload.sh web`, silently clearing daily caps such as `RATE_LIMIT_MEETING_EXTRACT_PER_DAY`. It also means the pre-2026-07-28 get-then-set race (§0o) was a **live production** bypass, not a dev-only concern. **The operator must still set `REDIS_URL` in the live `.env`** — a gitignored file no PR can reach. | `.env.example`, live `.env` | doc fixed; operator action pending | + +--- + ## 0. MVP-complete open items (2026-04-29, operator action required) These are not blocking pilot but must be resolved before a second contributor joins or the platform is advertised publicly. From f62895bc950d25445441a6e2663c5b32986a87f0 Mon Sep 17 00:00:00 2001 From: bwerapol Date: Tue, 28 Jul 2026 21:08:43 +0000 Subject: [PATCH 2/3] docs(core): pre-register generation-uplift suite 2 (corpus-dependent, live-KRA) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Committed BEFORE any run, per the honesty guardrail suite 1 set: this file's first commit is the pre-registration timestamp. Task list, injection procedure and metric are locked as of this commit. Suite 1 measured the injection MECHANISM: generic utility tasks with a hand-written injected block. Four of its six tasks tied because the base model already knew the edge case. Suite 2 changes both variables — tasks whose correct answer is only derivable from BrainPlatform convention, and a treatment block taken verbatim from the LIVE KRA path rather than authored. That makes it a test of retrieval + generation together, i.e. the product. Confound control: KNOWN_ISSUES §0p / #174 established that scope:"user" knowledge is invisible outside its capture project. All five rules under test were taught into the canonical Brain Platform project and verified retrievable from a projectName:"BrainPlatform" session before this file was written — three of them ranking #1 for their own task query. Without that control a null could not distinguish "knowledge didn't help" from "knowledge wasn't visible". Records two caveats up front rather than discovering them later: grading is static assertion over emitted source (weaker than suite 1's executable tests, since these conventions govern code shape), and the control arm's repo isolation is instruction-enforced rather than sandboxed — which would inflate control and understate uplift, so any control pass is checked against its transcript and a contaminated task is reported void rather than as a tie. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01Rqj5y9fT3XPUqnqKyustp9 --- .../core/generation-uplift/suite-2/README.md | 127 ++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 packages/core/generation-uplift/suite-2/README.md diff --git a/packages/core/generation-uplift/suite-2/README.md b/packages/core/generation-uplift/suite-2/README.md new file mode 100644 index 0000000..2f753d7 --- /dev/null +++ b/packages/core/generation-uplift/suite-2/README.md @@ -0,0 +1,127 @@ +# Generation-uplift suite 2 — corpus-dependent, live-KRA injection + +Follow-up to suite 1 (`../README.md`, issue #126). Suite 1 answered *does knowing +a non-obvious convention up front change output?* using generic utility tasks and +a **hand-written** injected block. It found +33.3pp (control 4/6 → treatment 6/6), +but four of six tasks tied because the base model already knew the edge case from +training — the suite was corpus-*independent* by design, to avoid testing the +Brain with knowledge the same session had taught it. + +This suite changes the two things that made suite 1 a mechanism test rather than a +product test. + +## Pre-registration + +**Written and committed before any run. This file's first commit is the +pre-registration timestamp. Do not edit the task list, the injection procedure, or +the metric after runs start** — per the honesty guardrails in +`docs/VALIDATION.md §2` and the precedent set by suite 1. + +### What changes from suite 1 + +| | Suite 1 | Suite 2 | +|---|---|---| +| Task knowledge | corpus-independent (generic utilities) | **corpus-dependent** — only derivable from BrainPlatform convention | +| Treatment block | hand-written `injected-knowledge.md` | **live KRA output**, captured verbatim | +| What it measures | the injection mechanism | **retrieval + generation together** (the product) | +| Grading | runtime tests (`node:assert`) | **static assertions over emitted source** (see caveat) | + +### Design + +Five tasks. Each asks for a small, self-contained file of the kind this repo +actually contains. Each has exactly one **non-obvious BrainPlatform convention** +that a competent TypeScript/Next.js developer would plausibly get wrong, because +the idiomatic general answer differs from this repo's required answer. + +Only the task prompt is shown. Two measures keep the control arm from discovering +the convention by other means: + +1. **Prompts are framed generically** — they describe the artifact wanted + ("a Next.js server component that reads an env var and renders it") and never + name this repo, its files, or its conventions. A control agent has no cue that + a project-specific rule exists, so no reason to go looking for one. +2. **File reading is forbidden in the prompt**, and each arm writes its single + output file to a scratch directory outside the repo. + +**Enforcement caveat, stated up front.** Measure 2 is an *instruction*, not a +sandbox: the agent harness available here runs with a working directory inside +this repository, so a determined agent could read `docs/GUIDELINES.md` and find +the answer. Suite 1 described its arms as running with "no access to this repo"; +that was also instruction-enforced, and it mattered less there because suite 1's +tasks were generic utilities whose answers are not written down here. For a +corpus-*dependent* suite the risk is real and asymmetric: it would inflate the +**control** arm and therefore **understate** uplift. Any control-arm pass is +consequently checked against its transcript for evidence of file reads, and a +task whose control arm read repo files is reported as **void**, not as a tie. + +### Procedure + +For each task, run the same agent twice: + +- **Control arm:** task prompt only. +- **Treatment arm:** task prompt + the knowledge block returned by + `brain_start_session({ prompt: , projectName: + "BrainPlatform" })`, pasted verbatim under a "Relevant knowledge from your + Brain" heading. + +The treatment block is **not authored** — whatever KRA returns is what the agent +gets, including irrelevant items. If KRA returns nothing useful for a task, that +task is expected to tie, and that tie is a real result about retrieval, not a +flaw in the harness. + +Same model both arms. Same task prompt text both arms; the injected block is the +single deliberate variable. Seed/temperature are not controllable through the +available agent harness — a known limitation, recorded rather than hidden. + +### Confound control (why these five tasks) + +`KNOWN_ISSUES §0p` documents that `scope: "user"` knowledge is invisible outside +its capture project (117 rows; issue #174). Every rule below was **taught into the +canonical `Brain Platform` project on 2026-07-28 and verified retrievable** from a +`projectName: "BrainPlatform"` session before this file was written. Without that +control, a null result could not distinguish "the knowledge didn't help" from +"the knowledge wasn't visible" — which is exactly the ambiguity that makes an +uncontrolled live-KRA benchmark unpublishable. + +### Task list (pre-registered) + +| # | Task | Required convention | The plausible wrong answer | +|---|---|---|---| +| 1 | `app/status/page.tsx` — server component rendering `process.env.BRAIN_PUBLIC_HOSTNAME` | `export const dynamic = "force-dynamic"` | omit it; Docker build bakes the empty value | +| 2 | route handler redirecting `/old` → `/new` absolutely | path-only `Location`, or `x-forwarded-host` + `x-forwarded-proto` | `new URL("/new", req.url)` → emits `0.0.0.0:3000` | +| 3 | client component displaying the current hostname | SSR-safe default, real read in `useEffect` | read `window.location` in render / `useMemo` / `useState(init)` → React #418 | +| 4 | session card rendering a relative timestamp | import `formatRelative` from `@brain/core/format-relative` + hydration-safe wrapper | hand-roll a "N days ago" helper | +| 5 | i18n dictionary entry for a retrieved/cited count | format-string substitution or conditional trailing key | bake a sample count (`"0 items retrieved · 2 cited"`) into the string | + +### Metric (pre-registered) + +Per-task binary pass/fail on the convention assertion, control vs treatment, +n=5 per arm. Report the paired difference and the full pass/fail matrix, +including which tasks tied and why. At n=5 no confidence interval is meaningful; +this is reported as a small-n indicative read, consistent with suite 1 and with +the retrieval benchmark's own "n is small" caveat. + +**Pre-committed reporting rule:** the result is published either way, including a +null or a negative. A tie is reported as a tie, and the retrieved block that +produced it is included verbatim in `RESULTS.md` so a reader can judge whether +retrieval or generation was at fault. + +### Honesty caveats (read before trusting any number) + +1. **Grading is static, not runtime.** These conventions govern code *shape* + (a directive is present, a read sits inside an effect, an import comes from a + specific subpath) — there is no runtime behaviour to assert without booting + Next.js and a Docker build. Assertions are mechanical regex/AST checks over the + emitted file, so there is still no human or LLM judgement in the loop, but this + is a **weaker instrument** than suite 1's executable tests. Stated here rather + than discovered later. +2. **The author of the tasks also authored the backfilled knowledge** (2026-07-28, + same session). Mitigated by taking the rules verbatim from pre-existing repo + docs (`GUIDELINES §9/§10`, `AGENTS.md`) that long predate this suite — the + conventions are not invented for the benchmark — but the *selection* of which + five to test is mine, and selection is a bias surface. +3. **n=5.** Indicative, not powered. +4. **Live KRA output is not reproducible over time.** The corpus changes; decay + and usage counts move rankings. Each run's retrieved block is committed to + `RESULTS.md` so the read is auditable even though it is not re-runnable to the + same input. From 94b471f49caf3b5896bae236dfd7c05e159a498c Mon Sep 17 00:00:00 2001 From: bwerapol Date: Tue, 28 Jul 2026 21:13:31 +0000 Subject: [PATCH 3/3] =?UTF-8?q?feat(core):=20suite-2=20generation-uplift?= =?UTF-8?q?=20read=20=E2=80=94=20+40pp,=20and=20where=20injection=20pays?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Runs the suite pre-registered in f62895b: five corpus-dependent tasks, treatment arm injected verbatim from the LIVE KRA path. Control 3/5, treatment 5/5 — +40pp, 0 regressions. The aggregate is the less interesting half. What separates a flip from a tie is whether the convention is locally arbitrary: - FLIPS. Task 1 (force-dynamic) — control emitted no static-rendering opt-out of any kind, reproducing the v0.14.0 bug verbatim; the rule only exists because deploy/Dockerfile builds with dummy env. Task 4 (@brain/core/format-relative) — control hand-rolled its own formatter, exactly the divergence v0.15.0 consolidated away. Neither is derivable from general expertise. - TIES. Tasks 2, 3 and 5 are conventions that coincide with good practice: the control arm reached for x-forwarded-host unprompted, mount-gated its window read, and used {count} placeholders without being told. So injection pays where the convention is arbitrary and ties where it is craft — which predicts where capture effort returns most. Suite 1's ties had the same cause; two independent suites now agree. Caveats recorded rather than buried: n=5; grading is static assertion over emitted source; isolation was instruction-enforced and every control pass was checked for contamination (none found). Most importantly, the five rules were verified retrievable from a project-scoped session first, to control for #174 — roughly half the corpus is invisible to such a session today, so an uncurated run would score lower. This measures the Brain with its known retrieval defect controlled for, not as a user experiences it. Both arms' raw outputs committed under suite-2/tasks/ for audit. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01Rqj5y9fT3XPUqnqKyustp9 --- docs/VALIDATION.md | 47 +++++++ .../core/generation-uplift/suite-2/RESULTS.md | 99 +++++++++++++++ .../tasks/1-force-dynamic/control/page.tsx | 9 ++ .../tasks/1-force-dynamic/treatment/page.tsx | 10 ++ .../tasks/2-forwarded-host/control/route.ts | 37 ++++++ .../tasks/2-forwarded-host/treatment/route.ts | 23 ++++ .../3-mount-gate/control/hostname-badge.tsx | 33 +++++ .../3-mount-gate/treatment/hostname-badge.tsx | 17 +++ .../control/session-card.tsx | 61 +++++++++ .../treatment/session-card.tsx | 53 ++++++++ .../tasks/5-i18n-counts/control/i18n-entry.ts | 70 +++++++++++ .../5-i18n-counts/treatment/i18n-entry.ts | 117 ++++++++++++++++++ 12 files changed, 576 insertions(+) create mode 100644 packages/core/generation-uplift/suite-2/RESULTS.md create mode 100644 packages/core/generation-uplift/suite-2/tasks/1-force-dynamic/control/page.tsx create mode 100644 packages/core/generation-uplift/suite-2/tasks/1-force-dynamic/treatment/page.tsx create mode 100644 packages/core/generation-uplift/suite-2/tasks/2-forwarded-host/control/route.ts create mode 100644 packages/core/generation-uplift/suite-2/tasks/2-forwarded-host/treatment/route.ts create mode 100644 packages/core/generation-uplift/suite-2/tasks/3-mount-gate/control/hostname-badge.tsx create mode 100644 packages/core/generation-uplift/suite-2/tasks/3-mount-gate/treatment/hostname-badge.tsx create mode 100644 packages/core/generation-uplift/suite-2/tasks/4-format-relative/control/session-card.tsx create mode 100644 packages/core/generation-uplift/suite-2/tasks/4-format-relative/treatment/session-card.tsx create mode 100644 packages/core/generation-uplift/suite-2/tasks/5-i18n-counts/control/i18n-entry.ts create mode 100644 packages/core/generation-uplift/suite-2/tasks/5-i18n-counts/treatment/i18n-entry.ts diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index 68e26fa..3e0531e 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -19,6 +19,13 @@ regressions). Positive, but n=6 is small and the task suite under-tests "well-known" utilities — see `packages/core/generation-uplift/RESULTS.md` for the full honest read, including where injection made no difference. +- ✅ **Suite 2 — corpus-dependent tasks, live-KRA injection (2026-07-28, + below)**: +40pp (control 3/5, treatment 5/5, n=5, 0 regressions). More + importantly it identifies *where* injection pays: the two flips were + **locally arbitrary** conventions (a workspace subpath; a build-pipeline + quirk), and all three ties were conventions that coincide with general good + practice the base model already had. See + `packages/core/generation-uplift/suite-2/RESULTS.md`. ## First published run — retrieval NDCG@5 (2026-07-06) @@ -87,6 +94,46 @@ before any run) and `RESULTS.md` (the read). Summary: including a grading-mechanics substitution (no local vitest available; `harness/grade.ts` re-implements the same assertions with `node:assert`). +## Second published run — suite 2, corpus-dependent + live KRA (2026-07-28) + +Suite 1 measured the injection *mechanism*: generic utility tasks with a +hand-written injected block. Suite 2 changes both variables — tasks whose correct +answer is only derivable from BrainPlatform convention, and a treatment block +taken **verbatim from the live KRA path** rather than authored. That makes it a +test of retrieval and generation together. Pre-registration committed before any +run (`packages/core/generation-uplift/suite-2/README.md`). + +| | Convention applied (n=5) | +|---|---| +| Control | 3/5 (60%) | +| **Treatment (live-KRA)** | **5/5 (100%)** | +| Paired difference | **+40pp, 0 regressions** | + +- **The aggregate is not the interesting part.** The two flips were conventions + that are *locally arbitrary* and cannot be derived from expertise: the + `@brain/core/format-relative` subpath (control hand-rolled its own formatter — + exactly the divergence v0.15.0 consolidated away), and `force-dynamic` on an + env-reading server component (control emitted **no** static-rendering opt-out of + any kind — the v0.14.0 bug verbatim). All three ties were conventions that + coincide with general good practice: the control arm reached for + `x-forwarded-host` unprompted, mount-gated its `window` read, and used `{count}` + placeholders without being told. +- **So injection pays off where the convention is arbitrary, and ties where it is + good craft.** That predicts where capture effort has the highest return — local + arbitrariness (package paths, build-pipeline quirks, project decisions), not + general engineering practice. Suite 1's ties had the same cause, so two + independent suites now agree on the mechanism. +- **Reading it honestly:** n=5; grading is static assertion over emitted source + (weaker than suite 1's executable tests, pre-registered as such); isolation was + instruction-enforced and every control pass was checked for contamination (none + found — no repo identifiers in any control output). **Most important caveat:** + the five rules were verified retrievable from a project-scoped session *before* + the run, to control for the scope-filter defect in `KNOWN_ISSUES §0p` / #174. + Roughly half the corpus is invisible to a project-scoped session until that is + resolved, so **a run against the uncurated corpus would score lower.** This + number describes the Brain with its known retrieval defect controlled for, not + the Brain as a user experiences it today. + ### The earlier attempt, and why the label changed A previous pair of scripts ran against the dev seed corpus diff --git a/packages/core/generation-uplift/suite-2/RESULTS.md b/packages/core/generation-uplift/suite-2/RESULTS.md new file mode 100644 index 0000000..9a3c2e6 --- /dev/null +++ b/packages/core/generation-uplift/suite-2/RESULTS.md @@ -0,0 +1,99 @@ +# Suite 2 results — first read (2026-07-28) + +Run against the pre-registration in `README.md` (committed `f62895b`, before any +arm was run). Task list, injection procedure and metric unchanged since. + +## Headline + +| | Convention applied (n=5) | +|---|---| +| Control (no injected knowledge) | 3/5 (60%) | +| Treatment (live-KRA injection) | **5/5 (100%)** | +| Paired difference | **+40pp, 0 regressions** | + +## Per-task matrix + +| # | Convention under test | Control | Treatment | | +|---|---|---|---|---| +| 1 | `export const dynamic = "force-dynamic"` on an env-reading server component | ❌ | ✅ | **flip** | +| 2 | `x-forwarded-host` / path-only `Location` instead of `req.url` | ✅ | ✅ | tie | +| 3 | Mount-gate `window.location` behind `useEffect` | ✅ | ✅ | tie | +| 4 | Import `formatRelative` from `@brain/core/format-relative` | ❌ | ✅ | **flip** | +| 5 | No baked numbers in i18n dictionary strings | ✅ | ✅ | tie | + +## What actually separates a flip from a tie + +This is the finding, and it is sharper than suite 1's aggregate number. + +**The two flips are conventions that are locally arbitrary.** Neither is +derivable from general expertise: + +- **Task 1** — `force-dynamic` is only required here because `deploy/Dockerfile` + builds with dummy env vars, so Next.js pre-renders the page with empty values. + A competent Next.js developer writing this page in the abstract has no reason to + add the directive. The control output contained **no static-rendering opt-out of + any kind** (checked for `dynamic`, `revalidate`, `fetchCache`, `unstable_noStore`) + — this is the exact bug that shipped in v0.14.0 and needed two fix rounds. +- **Task 4** — `@brain/core/format-relative` is a workspace subpath. It cannot be + guessed. The control arm hand-rolled its own `formatRelative(from, now, locale)` + — a perfectly reasonable implementation, and precisely the divergence v0.15.0 + consolidated away after four local formatters drifted apart. + +**The three ties are conventions that coincide with general good practice**, which +the base model already had: + +- **Task 2** — the control arm reached for `x-forwarded-host` unprompted and even + wrote a comment explaining that `request.url` yields `http://localhost:3000` + inside a container. Standard reverse-proxy knowledge. +- **Task 3** — the control arm used `useState(null)` + `useEffect`, the textbook + SSR-safe pattern. +- **Task 5** — the control arm used `{count}` placeholders and pluralisation keys + without being told. + +**So: injected knowledge paid off exactly where the convention is arbitrary — a +package path, a build-pipeline quirk — and tied everywhere the convention is +something a good engineer already does.** That is a more useful statement of what +this Brain is for than the aggregate percentage. It also predicts where future +capture effort has the highest return: local arbitrariness, not general craft. + +Suite 1 saw the same shape for the same reason (its four ties were edge cases the +base model knew from training). Two independent suites now agree on the mechanism. + +## Reading it honestly + +- **n=5.** Indicative, not powered. No confidence interval is meaningful. +- **Grading is static.** Assertions are mechanical checks over the emitted source + (directive present; import path; `window` read inside an effect; no digit inside + a dictionary string) — no human or LLM judgement, but a **weaker instrument** + than suite 1's executable tests, because these conventions govern code shape and + have no runtime behaviour to assert without a full Next.js build. Pre-registered + as such, not discovered afterwards. +- **Isolation was instruction-enforced, not sandboxed.** Per the pre-registration, + every control pass was checked for contamination. No control output contains any + repo-specific identifier (`BrainPlatform`, `External Brain`, `useTweaks`, + `bp_tweaks`, `@brain/core`), and each agent used 1–2 tool calls, consistent with + writing its single output file. **No task is reported void.** The residual risk + is that contamination would inflate control and therefore *understate* the + uplift reported here. +- **Same-session authorship.** The five rules were backfilled into the Brain on + 2026-07-28, the same day this suite ran. Mitigated by taking each rule verbatim + from repo docs (`GUIDELINES §9/§10`, `AGENTS.md`) that long predate the suite — + the conventions are not invented for the benchmark — but *selecting* which five + to test is an author-bias surface, as is the fact that the same session both + taught and tested them. +- **Retrieval was real, but the corpus was curated.** The treatment blocks are + genuine live KRA output, including irrelevant items (task 4's block carried a + `KnowledgeItemView` rule that has nothing to do with timestamps). But the five + rules had been verified retrievable from a `projectName: "BrainPlatform"` session + first, precisely to control for the scope-filter defect in + `KNOWN_ISSUES §0p` / #174. **A run against the uncurated corpus would score + lower** — roughly half of it is invisible to a project-scoped session until + #174 is resolved. This number therefore describes the Brain *with its known + retrieval defect controlled for*, not the Brain as a user experiences it today. + +## Reproduction + +Task prompts, injected blocks (verbatim) and both arms' emitted files are the +inputs; the grading assertions are in the per-task table above. Live KRA output is +not reproducible over time — the corpus changes, decay and usage counts move +rankings — so the retrieved blocks are recorded here rather than regenerated. diff --git a/packages/core/generation-uplift/suite-2/tasks/1-force-dynamic/control/page.tsx b/packages/core/generation-uplift/suite-2/tasks/1-force-dynamic/control/page.tsx new file mode 100644 index 0000000..dbc55da --- /dev/null +++ b/packages/core/generation-uplift/suite-2/tasks/1-force-dynamic/control/page.tsx @@ -0,0 +1,9 @@ +const publicHostname = process.env.PUBLIC_HOSTNAME ?? 'localhost'; + +export default function Page() { + return ( +
+

{publicHostname}

+
+ ); +} diff --git a/packages/core/generation-uplift/suite-2/tasks/1-force-dynamic/treatment/page.tsx b/packages/core/generation-uplift/suite-2/tasks/1-force-dynamic/treatment/page.tsx new file mode 100644 index 0000000..9d93119 --- /dev/null +++ b/packages/core/generation-uplift/suite-2/tasks/1-force-dynamic/treatment/page.tsx @@ -0,0 +1,10 @@ +// Without force-dynamic, Next would prerender this page at `next build` — inside +// the Docker image build, where PUBLIC_HOSTNAME is not the runtime value — and +// bake the wrong hostname into the static HTML. +export const dynamic = "force-dynamic"; + +const publicHostname = process.env.PUBLIC_HOSTNAME ?? ""; + +export default function Page() { + return

{publicHostname}

; +} diff --git a/packages/core/generation-uplift/suite-2/tasks/2-forwarded-host/control/route.ts b/packages/core/generation-uplift/suite-2/tasks/2-forwarded-host/control/route.ts new file mode 100644 index 0000000..8b0a9dd --- /dev/null +++ b/packages/core/generation-uplift/suite-2/tasks/2-forwarded-host/control/route.ts @@ -0,0 +1,37 @@ +import { NextResponse, type NextRequest } from 'next/server'; + +export const dynamic = 'force-dynamic'; + +// Inside the container the app is reached over plain HTTP on an internal +// hostname, so `request.url` yields e.g. http://localhost:3000 — never the +// public origin. The TLS-terminating proxy is the only source of truth for the +// externally visible scheme/host, so read its X-Forwarded-* headers. These are +// trustworthy only because every request is forced through that proxy. +function externalOrigin(request: NextRequest): string { + const headers = request.headers; + + // A proxy chain appends to these, so the client-most value is first. + const forwardedHost = + first(headers.get('x-forwarded-host')) ?? headers.get('host'); + const forwardedProto = first(headers.get('x-forwarded-proto')); + + if (!forwardedHost) { + return request.nextUrl.origin; + } + + const proto = forwardedProto ?? request.nextUrl.protocol.replace(':', ''); + return `${proto}://${forwardedHost}`; +} + +function first(value: string | null): string | null { + if (!value) return null; + const head = value.split(',')[0]?.trim(); + return head ? head : null; +} + +export function GET(request: NextRequest): NextResponse { + const target = new URL('/new', externalOrigin(request)); + target.search = request.nextUrl.search; + + return NextResponse.redirect(target, 308); +} diff --git a/packages/core/generation-uplift/suite-2/tasks/2-forwarded-host/treatment/route.ts b/packages/core/generation-uplift/suite-2/tasks/2-forwarded-host/treatment/route.ts new file mode 100644 index 0000000..a519c0b --- /dev/null +++ b/packages/core/generation-uplift/suite-2/tasks/2-forwarded-host/treatment/route.ts @@ -0,0 +1,23 @@ +import { headers } from "next/headers"; +import { NextResponse } from "next/server"; + +export const dynamic = "force-dynamic"; + +export async function GET() { + const h = await headers(); + + // Inside the container the request is plain HTTP on an internal host, so + // req.url/req.nextUrl would build an absolute URL pointing at the container + // rather than the public origin. Only the proxy's forwarded headers know it. + const forwardedProto = h.get("x-forwarded-proto")?.split(",")[0]?.trim(); + const forwardedHost = h.get("x-forwarded-host")?.split(",")[0]?.trim(); + + const proto = forwardedProto || "https"; + const host = forwardedHost || h.get("host"); + + if (!host) { + return NextResponse.redirect("/new", 308); + } + + return NextResponse.redirect(`${proto}://${host}/new`, 308); +} diff --git a/packages/core/generation-uplift/suite-2/tasks/3-mount-gate/control/hostname-badge.tsx b/packages/core/generation-uplift/suite-2/tasks/3-mount-gate/control/hostname-badge.tsx new file mode 100644 index 0000000..a589269 --- /dev/null +++ b/packages/core/generation-uplift/suite-2/tasks/3-mount-gate/control/hostname-badge.tsx @@ -0,0 +1,33 @@ +'use client'; + +import { useEffect, useState } from 'react'; + +export function HostnameBadge() { + // window is undefined during SSR and on the hydration pass, so the hostname + // is read in an effect — reading it during render would desync the markup. + const [hostname, setHostname] = useState(null); + + useEffect(() => { + setHostname(window.location.hostname); + }, []); + + if (hostname === null) return null; + + return ( + + {hostname} + + ); +} + +export default HostnameBadge; diff --git a/packages/core/generation-uplift/suite-2/tasks/3-mount-gate/treatment/hostname-badge.tsx b/packages/core/generation-uplift/suite-2/tasks/3-mount-gate/treatment/hostname-badge.tsx new file mode 100644 index 0000000..258c12a --- /dev/null +++ b/packages/core/generation-uplift/suite-2/tasks/3-mount-gate/treatment/hostname-badge.tsx @@ -0,0 +1,17 @@ +'use client'; + +import { useEffect, useState } from 'react'; + +export function HostnameBadge() { + const [hostname, setHostname] = useState(null); + + useEffect(() => { + setHostname(window.location.hostname); + }, []); + + return ( + + {hostname ?? '—'} + + ); +} diff --git a/packages/core/generation-uplift/suite-2/tasks/4-format-relative/control/session-card.tsx b/packages/core/generation-uplift/suite-2/tasks/4-format-relative/control/session-card.tsx new file mode 100644 index 0000000..6691b81 --- /dev/null +++ b/packages/core/generation-uplift/suite-2/tasks/4-format-relative/control/session-card.tsx @@ -0,0 +1,61 @@ +// Server Component on purpose: the relative label is computed once during the +// server render and shipped as static HTML. A client component computing +// `Date.now()` during render would hydrate against a different clock and +// produce a mismatch. + +const MINUTE = 60; +const HOUR = 60 * MINUTE; +const DAY = 24 * HOUR; +const WEEK = 7 * DAY; +const MONTH = 30 * DAY; +const YEAR = 365 * DAY; + +const RELATIVE_UNITS: ReadonlyArray = [ + [YEAR, 'year'], + [MONTH, 'month'], + [WEEK, 'week'], + [DAY, 'day'], + [HOUR, 'hour'], + [MINUTE, 'minute'], + [1, 'second'], +]; + +function formatRelative(from: Date, now: Date, locale?: string): string { + const seconds = Math.round((from.getTime() - now.getTime()) / 1000); + const magnitude = Math.abs(seconds); + + if (magnitude < 45) return 'just now'; + + const formatter = new Intl.RelativeTimeFormat(locale, { numeric: 'auto' }); + const [size, unit] = RELATIVE_UNITS.find(([threshold]) => magnitude >= threshold) ?? [1, 'second']; + + return formatter.format(Math.round(seconds / size), unit); +} + +export interface SessionCardProps { + title: string; + /** ISO-8601 timestamp, e.g. "2026-07-26T09:12:00.000Z". */ + startedAt: string; + /** BCP-47 tag; defaults to the runtime locale. */ + locale?: string; +} + +export function SessionCard({ title, startedAt, locale }: SessionCardProps) { + const started = new Date(startedAt); + const valid = !Number.isNaN(started.getTime()); + + return ( +
+

{title}

+ {valid ? ( + + ) : ( + Unknown date + )} +
+ ); +} + +export default SessionCard; diff --git a/packages/core/generation-uplift/suite-2/tasks/4-format-relative/treatment/session-card.tsx b/packages/core/generation-uplift/suite-2/tasks/4-format-relative/treatment/session-card.tsx new file mode 100644 index 0000000..4dfefb9 --- /dev/null +++ b/packages/core/generation-uplift/suite-2/tasks/4-format-relative/treatment/session-card.tsx @@ -0,0 +1,53 @@ +'use client'; + +import { useEffect, useState } from 'react'; +import { formatRelative } from '@brain/core/format-relative'; + +const ABSOLUTE = new Intl.DateTimeFormat('en-US', { + year: 'numeric', + month: 'short', + day: 'numeric', + timeZone: 'UTC', +}); + +function formatAbsolute(iso: string): string { + const date = new Date(iso); + return Number.isNaN(date.getTime()) ? '' : ABSOLUTE.format(date); +} + +interface RelativeTimeProps { + iso: string; + className?: string; +} + +export function RelativeTime({ iso, className }: RelativeTimeProps) { + const absolute = formatAbsolute(iso); + const [label, setLabel] = useState(absolute); + + useEffect(() => { + setLabel(formatRelative(iso)); + }, [iso]); + + return ( + + ); +} + +export interface SessionCardProps { + title: string; + updatedAt: string; + className?: string; +} + +export function SessionCard({ title, updatedAt, className }: SessionCardProps) { + return ( +
+

{title}

+ +
+ ); +} + +export default SessionCard; diff --git a/packages/core/generation-uplift/suite-2/tasks/5-i18n-counts/control/i18n-entry.ts b/packages/core/generation-uplift/suite-2/tasks/5-i18n-counts/control/i18n-entry.ts new file mode 100644 index 0000000..214e745 --- /dev/null +++ b/packages/core/generation-uplift/suite-2/tasks/5-i18n-counts/control/i18n-entry.ts @@ -0,0 +1,70 @@ +/** + * Oracle status line — "N items retrieved · M cited" + * + * Dictionary entries live in the centralised per-locale dictionary object. + * Only the English entries are given here; other locales copy the same keys. + */ + +// --------------------------------------------------------------------------- +// Dictionary entries (en) +// --------------------------------------------------------------------------- + +export const en = { + // ...existing keys + + /** Singular / plural halves are separate keys so locales can diverge. */ + 'oracle.status.retrieved.one': '{count} item retrieved', + 'oracle.status.retrieved.other': '{count} items retrieved', + 'oracle.status.cited': '{count} cited', + + /** Joins the two halves; the separator is part of the string so RTL and + * locales that prefer a comma can override it. */ + 'oracle.status.line': '{retrieved} · {cited}', + + /** Shown when retrieval returned nothing at all. */ + 'oracle.status.empty': 'No knowledge retrieved', + + // ...existing keys +} as const; + +// --------------------------------------------------------------------------- +// Rendering with real counts +// --------------------------------------------------------------------------- + +type Dict = typeof en; +type Key = keyof Dict; + +/** Whatever the app already uses — shown here for completeness. */ +declare function t( + key: K, + vars?: Record, +): string; + +export function OracleStatusLine({ + retrievedCount, + citedCount, +}: { + retrievedCount: number; + citedCount: number; +}) { + if (retrievedCount === 0) { + return

{t('oracle.status.empty')}

; + } + + const retrieved = t( + retrievedCount === 1 + ? 'oracle.status.retrieved.one' + : 'oracle.status.retrieved.other', + { count: retrievedCount }, + ); + const cited = t('oracle.status.cited', { count: citedCount }); + + return ( +

+ {t('oracle.status.line', { retrieved, cited })} +

+ ); +} + +// OracleStatusLine({ retrievedCount: 5, citedCount: 2 }) -> "5 items retrieved · 2 cited" +// OracleStatusLine({ retrievedCount: 1, citedCount: 0 }) -> "1 item retrieved · 0 cited" diff --git a/packages/core/generation-uplift/suite-2/tasks/5-i18n-counts/treatment/i18n-entry.ts b/packages/core/generation-uplift/suite-2/tasks/5-i18n-counts/treatment/i18n-entry.ts new file mode 100644 index 0000000..c4f67fd --- /dev/null +++ b/packages/core/generation-uplift/suite-2/tasks/5-i18n-counts/treatment/i18n-entry.ts @@ -0,0 +1,117 @@ +/** + * Oracle retrieval status line — "5 items retrieved · 2 cited". + * + * The counts are NOT baked into the dictionary values: each string carries a + * `{n}` placeholder that is substituted at render time. A dictionary entry has + * to stay correct for every possible value of the data behind it, so a literal + * number in a locale string is always a bug waiting to happen. + */ + +// --------------------------------------------------------------------------- +// Dictionary entries (English) +// --------------------------------------------------------------------------- + +export const en = { + // Retrieval count. Two plural forms so "1 item retrieved" reads correctly. + 'oracle.status.retrieved_one': '{n} item retrieved', + 'oracle.status.retrieved_other': '{n} items retrieved', + + // Citation count — a SEPARATE key, rendered conditionally, so the segment can + // be omitted entirely when nothing was cited (rather than showing "0 cited"). + 'oracle.status.cited': '{n} cited', + + // Separator lives in the dictionary too: some locales prefer a comma or a + // full-width middle dot over "·". + 'oracle.status.separator': ' · ', + + // Shown instead of the whole line when retrieval returned nothing. + 'oracle.status.empty': 'No knowledge retrieved', +} as const; + +export type OracleStatusKey = keyof typeof en; + +// --------------------------------------------------------------------------- +// Interpolation helper (the generic one your `t()` already exposes) +// --------------------------------------------------------------------------- + +type Vars = Record; + +export function t( + dict: Record, + key: string, + vars?: Vars, +): string { + const template = dict[key] ?? key; + if (!vars) return template; + return template.replace(/\{(\w+)\}/g, (match, name: string) => + name in vars ? String(vars[name]) : match, + ); +} + +/** Picks the plural variant, then interpolates. */ +function tPlural( + dict: Record, + baseKey: string, + n: number, +): string { + const form = new Intl.PluralRules('en').select(n); // 'one' | 'other' + const key = `${baseKey}_${form}`; + return t(dict, key in dict ? key : `${baseKey}_other`, { n }); +} + +// --------------------------------------------------------------------------- +// Component usage +// --------------------------------------------------------------------------- + +/** + * Composes the line from real counts: "5 items retrieved · 2 cited", or just + * "5 items retrieved" when nothing was cited, or the empty-state string when + * nothing was retrieved. The cited segment is a conditional append, not a + * hard-coded tail on the retrieved string. + */ +export function oracleStatusLine( + dict: Record, + retrievedCount: number, + citedCount: number, +): string { + if (retrievedCount === 0) return t(dict, 'oracle.status.empty'); + + const retrieved = tPlural(dict, 'oracle.status.retrieved', retrievedCount); + if (citedCount === 0) return retrieved; + + return ( + retrieved + + t(dict, 'oracle.status.separator') + + t(dict, 'oracle.status.cited', { n: citedCount }) + ); +} + +// oracleStatusLine(en, 5, 2) -> "5 items retrieved · 2 cited" +// oracleStatusLine(en, 1, 0) -> "1 item retrieved" +// oracleStatusLine(en, 0, 0) -> "No knowledge retrieved" + +/* + * Component usage (lives in a .tsx file): + * + * export function OracleStatusLine({ retrievedCount, citedCount }: Props) { + * const dict = useDictionary(); + * return ( + *

+ * {oracleStatusLine(dict, retrievedCount, citedCount)} + *

+ * ); + * } + * + * If the cited segment needs its own markup (e.g. a link to the citations), + * render the pieces instead of the joined string — same keys, same guard: + * + * {tPlural(dict, 'oracle.status.retrieved', retrievedCount)} + * {citedCount > 0 && ( + * <> + * {t(dict, 'oracle.status.separator')} + * + * + * )} + */