diff --git a/.github/benchmark-site/coverage/index.html b/.github/benchmark-site/coverage/index.html new file mode 100644 index 000000000..e25085e3e --- /dev/null +++ b/.github/benchmark-site/coverage/index.html @@ -0,0 +1,148 @@ + + + + + + + + Harness stack coverage + + + + + + + + +
+ + + iii + Harness benchmarks + + + +
+ +
+
+
+
+ + Harness stack +
+

Code coverage

+

LLVM line coverage of the instrumented stack, refreshed by the daily E2E cycle.

+
+
+ Last published + +
+
+ + + + +
+ + + + + + diff --git a/.github/benchmark-site/execution-data.js b/.github/benchmark-site/execution-data.js index 5ec8c1b7c..9d70003e3 100644 --- a/.github/benchmark-site/execution-data.js +++ b/.github/benchmark-site/execution-data.js @@ -852,6 +852,42 @@ return history?.executions?.find((execution) => execution.id === id) || null; } + function groupRunFailures(reports) { + const groups = []; + (Array.isArray(reports) ? reports : []).forEach((record) => { + (record?.report?.scenarios || []).forEach((scenario) => { + (scenario.runs || []).forEach((run, runIndex) => { + const items = []; + (run.failures || []).forEach((failure) => { + items.push({ + kind: "failure", + phase: failure.phase || "failure", + message: failure.message || "No failure message", + }); + }); + (run.hard_gates || []) + .filter((gate) => gate && gate.passed === false) + .forEach((gate) => { + items.push({ + kind: "gate", + gateId: gate.id, + message: gate.reason || "Hard gate failed", + }); + }); + if (items.length) { + groups.push({ + subjectId: record.subject_id, + scenarioId: scenario.scenario_id, + runIndex, + items, + }); + } + }); + }); + }); + return groups; + } + return { buildEfficiencyOverview, contractFingerprint, @@ -859,6 +895,7 @@ executionsWithinDays, filterExecutions, findExecution, + groupRunFailures, legacyExecution, matrixCell, matrixCellLabel, diff --git a/.github/benchmark-site/execution-data.test.cjs b/.github/benchmark-site/execution-data.test.cjs index ea1997012..405c2f542 100644 --- a/.github/benchmark-site/execution-data.test.cjs +++ b/.github/benchmark-site/execution-data.test.cjs @@ -8,6 +8,7 @@ const { executionsWithinDays, filterExecutions, findExecution, + groupRunFailures, matrixCell, matrixCellLabel, matrixRows, @@ -514,3 +515,64 @@ test("compares efficiency only within the same scenario contract", () => { assert.equal(overview.metrics.tokens.comparableBaseline, 100); assert.equal(overview.metrics.tokens.delta, -10); }); + +test("groupRunFailures returns empty for missing or empty reports", () => { + assert.deepEqual(groupRunFailures(undefined), []); + assert.deepEqual(groupRunFailures([]), []); + assert.deepEqual( + groupRunFailures([ + { + subject_id: "s", + report: { + scenarios: [ + { + scenario_id: "clean", + runs: [{ failures: [], hard_gates: [{ id: "g", passed: true }] }], + }, + ], + }, + }, + ]), + [], + ); +}); + +test("groupRunFailures groups failures and failed gates per run", () => { + const groups = groupRunFailures([ + { + subject_id: "anthropic-sonnet", + report: { + scenarios: [ + { + scenario_id: "security_review", + runs: [ + { + failures: [{ phase: "execute", message: "boom" }], + hard_gates: [ + { id: "no_secrets", passed: false, reason: "leaked" }, + { id: "compiles", passed: true }, + ], + }, + { failures: [], hard_gates: [] }, + { failures: [{ message: "" }] }, + ], + }, + ], + }, + }, + ]); + assert.equal(groups.length, 2); + assert.deepEqual(groups[0], { + subjectId: "anthropic-sonnet", + scenarioId: "security_review", + runIndex: 0, + items: [ + { kind: "failure", phase: "execute", message: "boom" }, + { kind: "gate", gateId: "no_secrets", message: "leaked" }, + ], + }); + assert.equal(groups[1].runIndex, 2); + assert.deepEqual(groups[1].items, [ + { kind: "failure", phase: "failure", message: "No failure message" }, + ]); +}); diff --git a/.github/benchmark-site/execution.html b/.github/benchmark-site/execution.html index 114a871ba..d4d948cba 100644 --- a/.github/benchmark-site/execution.html +++ b/.github/benchmark-site/execution.html @@ -92,16 +92,12 @@

Execution failures

—
—
-
-
Reliability events
-
—
-
Blocking failures and missing reports
-
${renderConversationLaunch(messages, run)} -
- ${runSection("Failures", renderFailureList(run.failures))} - ${runSection("Hard gates", renderGateTable(run.hard_gates), { wide: true })} - ${runSection("Scored criteria", renderCriteriaTable(run.criteria), { wide: true })} - ${runSection("Usage and cost", usageBlocks(run), { wide: true })} - ${runSection("Retry attempts", renderRetries(run.retry_attempts), { wide: true })} - ${runSection("Prompt", `
${escapeHtml(run.prompt || "No prompt recorded.")}
`, { wide: true })} - ${runSection("Sessions and traces", sessionTable(run.metrics), { wide: true })} - ${runSection( - "Complete run record", - `
Show JSON
${escapeHtml(
-            JSON.stringify(run, null, 2),
-          )}
`, - { wide: true }, - )} +
+ ${RUN_TABS.map( + (tab) => + ``, + ).join("")}
+ ${RUN_TABS.map( + (tab) => + ``, + ).join("")} `; container.querySelector(".conversation-open")?.addEventListener("click", () => { openConversationDialog(run, context); }); + const buttons = [...container.querySelectorAll(".run-tabs button")]; + const activateRunTab = (tabId) => { + RUN_TABS.forEach((tab) => { + const panel = container.querySelector(`[data-panel="${tab.id}"]`); + const active = tab.id === tabId; + if (active && !panel.dataset.rendered) { + panel.dataset.rendered = "true"; + tab.render(panel, run); + } + panel.hidden = !active; + }); + buttons.forEach((button) => { + const active = button.dataset.tab === tabId; + button.classList.toggle("active", active); + button.setAttribute("aria-selected", String(active)); + }); + }; + buttons.forEach((button) => { + button.addEventListener("click", () => activateRunTab(button.dataset.tab)); + }); + activateRunTab("evaluation"); } - function renderFullScenario(record) { + function renderFullScenario(record, options = {}) { const report = record.report; const scenario = report.scenarios?.find( (item) => item.scenario_id === record.scenario_id, @@ -804,15 +937,26 @@ const aggregate = scenario.aggregate || {}; const policy = scenario.execution_policy || {}; const anchor = scenarioAnchor(subjectId, scenario.scenario_id); + const failingRun = (scenario.runs || []).some( + (run) => + (run.failures || []).length || + (run.hard_gates || []).some((gate) => gate && gate.passed === false), + ); + const open = !scenario.passed || failingRun || options.soleScenario; return ` -
-
-
- ${escapeHtml(report.subject.provider)}/${escapeHtml(report.subject.model)} -

${escapeHtml(titleCase(scenario.scenario_id))}

-
- ${meta.label} -
+
+ ${scenarioSummaryRow({ + title: titleCase(scenario.scenario_id), + subjectLabel: `${report.subject.provider}/${report.subject.model}`, + median: `${compactNumber(aggregate.median_score, 1)} / ${compactNumber(scenario.threshold, 0)}`, + passRate: formatPercent( + typeof aggregate.pass_rate === "number" ? aggregate.pass_rate * 100 : null, + ), + cost: formatCurrency(aggregate.cost?.total_usd), + statusCss: meta.css, + statusLabel: meta.label, + })} +
${metricBlock("Median score", compactNumber(aggregate.median_score, 1), `target ${compactNumber(scenario.threshold, 0)}`)} ${metricBlock("Pass rate", formatPercent( @@ -860,7 +1004,8 @@ }) .join("")}
-
+
+ `; } @@ -910,9 +1055,11 @@ ); elements.scenarioIntro.textContent = `${availableReports.length} scenario reports and ${runCount} individual runs. ` + - "Expand a run to render its complete diagnostics."; + "Select a scenario to expand its runs."; elements.scenarioDetails.innerHTML = availableReports - .map(renderFullScenario) + .map((record) => + renderFullScenario(record, { soleScenario: availableReports.length === 1 }), + ) .join(""); attachRunRenderers(); return; @@ -921,10 +1068,16 @@ execution.availability === "aggregate" ? "The complete report expired after 30 executions. Historical aggregate metrics remain available." : "This workflow did not produce a complete benchmark report."; + const aggregateCount = execution.subjects.reduce( + (total, subject) => total + (subject.scenarios || []).length, + 0, + ); elements.scenarioDetails.innerHTML = execution.subjects .flatMap((subject) => (subject.scenarios || []).map((scenario) => - renderAggregateScenario(subject, scenario), + renderAggregateScenario(subject, scenario, { + soleScenario: aggregateCount === 1, + }), ), ) .join(""); @@ -934,58 +1087,54 @@ } } - function collectDetailedFailures() { - const failures = []; - (detail?.reports || []).forEach((record) => { - (record?.report?.scenarios || []).forEach((scenario) => { - (scenario.runs || []).forEach((run, index) => { - const anchor = runAnchor(record.subject_id, scenario.scenario_id, index); - (run.failures || []).forEach((failure) => { - failures.push({ - anchor, - label: `${titleCase(scenario.scenario_id)} · run ${index + 1}`, - message: `${titleCase(failure.phase)}: ${failure.message}`, - }); - }); - (run.hard_gates || []) - .filter((gate) => !gate.passed) - .forEach((gate) => { - failures.push({ - anchor, - label: `${titleCase(scenario.scenario_id)} · ${gate.id}`, - message: gate.reason, - }); - }); - }); - }); - }); - return failures; + const MAX_FAILURE_CHIPS = 5; + + function truncateText(value, limit) { + const text = String(value || ""); + return text.length > limit ? `${text.slice(0, limit - 1)}…` : text; + } + + function failureChip(group) { + const anchor = runAnchor(group.subjectId, group.scenarioId, group.runIndex); + const first = group.items[0]; + const preview = + first.kind === "gate" + ? `${first.gateId}: ${first.message}` + : `${titleCase(first.phase)}: ${first.message}`; + return ` + + ${group.items.length} issue${group.items.length === 1 ? "" : "s"} + ${escapeHtml(titleCase(group.scenarioId))} · run ${group.runIndex + 1} + ${escapeHtml(truncateText(preview, 140))} + + `; } function renderFailures() { - const failures = collectDetailedFailures(); + const groups = window.HarnessExecutionData.groupRunFailures(detail?.reports); const count = blockingFailures(); - if (!failures.length && count === 0 && execution.status === "passed") { + if (!groups.length && count === 0 && execution.status === "passed") { elements.failureBox.hidden = true; return; } elements.failureBox.hidden = false; - if (failures.length) { + elements.failureTitle.textContent = count + ? `Execution failures · ${count} blocking event${count === 1 ? "" : "s"}` + : "Execution failures"; + if (groups.length) { + const sorted = [...groups].sort((a, b) => b.items.length - a.items.length); + const visible = sorted.slice(0, MAX_FAILURE_CHIPS); + const overflow = sorted.slice(MAX_FAILURE_CHIPS); elements.failureSummary.innerHTML = ` - +
${visible.map(failureChip).join("")}
+ ${ + overflow.length + ? `
+ Show all ${sorted.length} failing runs +
${overflow.map(failureChip).join("")}
+
` + : "" + } `; return; } @@ -1001,8 +1150,13 @@ } function renderRawData() { - const raw = detail || execution; - elements.rawJson.textContent = JSON.stringify(raw, null, 2); + // The detail JSON can be multi-megabyte: serialize only when the preview + // is opened or the fallback download is clicked, never at page load. + elements.rawPreview.addEventListener("toggle", () => { + if (!elements.rawPreview.open || elements.rawPreview.dataset.rendered) return; + elements.rawPreview.dataset.rendered = "true"; + elements.rawJson.textContent = JSON.stringify(detail || execution, null, 2); + }); elements.rawActions.replaceChildren(); if (detail && execution.detail_path && !window.HARNESS_BENCHMARK_PREVIEW) { const link = document.createElement("a"); @@ -1013,14 +1167,19 @@ elements.rawActions.append(link); return; } - const blob = new Blob([JSON.stringify(raw, null, 2)], { - type: "application/json", - }); const link = document.createElement("a"); link.className = "button"; - link.href = URL.createObjectURL(blob); + link.href = "#raw-data"; link.download = `${execution.id}.json`; link.textContent = "Download available JSON"; + link.addEventListener("click", () => { + if (link.dataset.blobUrl) return; + const blob = new Blob([JSON.stringify(detail || execution, null, 2)], { + type: "application/json", + }); + link.dataset.blobUrl = URL.createObjectURL(blob); + link.href = link.dataset.blobUrl; + }); elements.rawActions.append(link); } @@ -1049,20 +1208,42 @@ return value; } + function revealAnchor(anchorId) { + if (!anchorId) return; + const target = document.getElementById(anchorId); + if (!target) return; + // Open the target and every collapsed ancestor so deep links land on + // rendered content; opening fires toggle, which drives the lazy renderers. + let node = target.closest("details"); + while (node) { + node.open = true; + node = node.parentElement?.closest("details"); + } + const disclosure = target.querySelector?.("details.section-disclosure"); + if (disclosure) disclosure.open = true; + requestAnimationFrame(() => target.scrollIntoView()); + } + + function attachAnchorNavigation() { + window.addEventListener("hashchange", () => { + revealAnchor(window.location.hash.slice(1)); + }); + // Same-hash re-clicks do not fire hashchange; handle triage links directly. + elements.failureSummary.addEventListener("click", (event) => { + const link = event.target.closest('a[href^="#"]'); + if (link) revealAnchor(link.getAttribute("href").slice(1)); + }); + } + function reveal() { elements.loading.hidden = true; elements.content.hidden = false; - requestAnimationFrame(() => { - const anchor = window.location.hash.slice(1); - if (!anchor) return; - const target = document.getElementById(anchor); - if (target?.tagName === "DETAILS") target.open = true; - requestAnimationFrame(() => target?.scrollIntoView()); - }); + requestAnimationFrame(() => revealAnchor(window.location.hash.slice(1))); } async function initialize() { attachTranscriptDialogControls(); + attachAnchorNavigation(); if (!execution) { elements.loading.hidden = true; elements.error.hidden = false; diff --git a/.github/benchmark-site/index.html b/.github/benchmark-site/index.html index 5bfe56027..de66c9bac 100644 --- a/.github/benchmark-site/index.html +++ b/.github/benchmark-site/index.html @@ -25,6 +25,7 @@