Skip to content

(MOT-4295) feat(ci): publish harness integration and e2e coverage to the metrics site #664

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ytallo merged 3 commits into from
Aug 1, 2026
Merged

(MOT-4295) feat(ci): publish harness integration and e2e coverage to the metrics site #664

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
713f635
feat(ci): publish harness integration and e2e coverage to the metrics…
ytallo Aug 1, 2026
a0b0b5b
refactor(benchmark-site): keep coverage summary on the dedicated land…
ytallo Aug 1, 2026
8b6cca6
refactor(benchmark-site): progressive-disclosure IA for the execution…
ytallo Aug 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
148 changes: 148 additions & 0 deletions .github/benchmark-site/coverage/index.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta
name="description"
content="LLVM line coverage for the harness integration and E2E suites."
>
<meta name="color-scheme" content="dark light">
<title>Harness stack coverage</title>
<link rel="stylesheet" href="../styles.css">
<style>
.coverage-grid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(18rem, 1fr));
gap: 1.25rem;
}
.coverage-card {
display: flex;
flex-direction: column;
gap: 0.75rem;
padding: 1.5rem;
}
.coverage-card h2 {
margin: 0;
}
.coverage-percent {
font-size: 2.5rem;
font-weight: 700;
font-variant-numeric: tabular-nums;
}
.coverage-meta {
opacity: 0.75;
font-size: 0.875rem;
}
.coverage-links {
display: flex;
gap: 0.75rem;
flex-wrap: wrap;
}
</style>
</head>
<body>
<a class="skip-link" href="#main">Skip to coverage reports</a>
<div class="ambient ambient-one" aria-hidden="true"></div>
<div class="ambient ambient-two" aria-hidden="true"></div>

<header class="topbar">
<a class="brand" href="https://github.com/iii-hq/workers" aria-label="iii workers">
<span class="brand-copy">
<strong>iii</strong>
<span>Harness benchmarks</span>
</span>
</a>
<nav class="topbar-actions" aria-label="Dashboard actions">
<a class="button button-secondary" href="../index.html">Executions</a>
<a class="button button-secondary" href="../coverage-trend/">Trend</a>
</nav>
</header>

<main id="main" class="page-shell overview-shell">
<section class="page-heading" aria-labelledby="page-title">
<div>
<div class="eyebrow">
<span class="live-dot" aria-hidden="true"></span>
Harness stack
</div>
<h1 id="page-title">Code coverage</h1>
<p>LLVM line coverage of the instrumented stack, refreshed by the daily E2E cycle.</p>
</div>
<div class="sync-block">
<span id="sync-label">Last published</span>
<time id="last-update" datetime="">Waiting for data</time>
</div>
</section>

<section id="empty-state" class="empty-state" hidden>
<div class="empty-icon" aria-hidden="true">⌁</div>
<h2>No coverage published yet</h2>
<p>The next scheduled Harness E2E Daily workflow will publish the first reports here.</p>
</section>

<section id="coverage-content" class="coverage-grid" hidden>
<article class="panel coverage-card" id="card-integration">
<h2>Integration suite</h2>
<div class="coverage-percent" data-percent>–</div>
<div class="coverage-meta" data-meta>Not published yet.</div>
<div class="coverage-links">
<a class="button button-secondary" href="./integration/index.html" data-report hidden>
Browse report
</a>
</div>
</article>
<article class="panel coverage-card" id="card-e2e">
<h2>E2E suite</h2>
<div class="coverage-percent" data-percent>–</div>
<div class="coverage-meta" data-meta>Not published yet.</div>
<div class="coverage-links">
<a class="button button-secondary" href="./e2e/index.html" data-report hidden>
Browse report
</a>
</div>
</article>
</section>
</main>

<script>
window.HARNESS_COVERAGE = null;
</script>
<script src="./summary.js"></script>
<script>
(function () {
var data = window.HARNESS_COVERAGE;
var empty = document.getElementById("empty-state");
var content = document.getElementById("coverage-content");
var suites = data && data.suites ? data.suites : {};
var hasAny = Boolean(suites.e2e || suites.integration);
if (!hasAny) {
empty.hidden = false;
return;
}
content.hidden = false;

if (data.updated_at) {
var time = document.getElementById("last-update");
time.dateTime = data.updated_at;
time.textContent = new Date(data.updated_at).toLocaleString();
}

function render(cardId, summary) {
var card = document.getElementById(cardId);
if (!summary || !summary.totals || !summary.totals.lines) return;
var lines = summary.totals.lines;
card.querySelector("[data-percent]").textContent =
lines.percent.toFixed(1) + "%";
card.querySelector("[data-meta]").textContent =
(lines.covered != null ? lines.covered.toLocaleString() : "?") +
" of " + (lines.count != null ? lines.count.toLocaleString() : "?") +
" lines covered · " + (summary.generated_at || "");
card.querySelector("[data-report]").hidden = false;
}
render("card-integration", suites.integration);
render("card-e2e", suites.e2e);
})();
</script>
</body>
</html>
37 changes: 37 additions & 0 deletions .github/benchmark-site/execution-data.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -852,13 +852,50 @@
return history?.executions?.find((execution) => execution.id === id) || null;
}

function groupRunFailures(reports) {
const groups = [];
(Array.isArray(reports) ? reports : []).forEach((record) => {
(record?.report?.scenarios || []).forEach((scenario) => {
(scenario.runs || []).forEach((run, runIndex) => {
const items = [];
(run.failures || []).forEach((failure) => {
items.push({
kind: "failure",
phase: failure.phase || "failure",
message: failure.message || "No failure message",
});
});
(run.hard_gates || [])
.filter((gate) => gate && gate.passed === false)
.forEach((gate) => {
items.push({
kind: "gate",
gateId: gate.id,
message: gate.reason || "Hard gate failed",
});
});
if (items.length) {
groups.push({
subjectId: record.subject_id,
scenarioId: scenario.scenario_id,
runIndex,
items,
});
}
});
});
});
return groups;
}

return {
buildEfficiencyOverview,
contractFingerprint,
executionEfficiencyTotalsFromDetail,
executionsWithinDays,
filterExecutions,
findExecution,
groupRunFailures,
legacyExecution,
matrixCell,
matrixCellLabel,
Expand Down
62 changes: 62 additions & 0 deletions .github/benchmark-site/execution-data.test.cjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ const {
executionsWithinDays,
filterExecutions,
findExecution,
groupRunFailures,
matrixCell,
matrixCellLabel,
matrixRows,
Expand Down Expand Up @@ -514,3 +515,64 @@ test("compares efficiency only within the same scenario contract", () => {
assert.equal(overview.metrics.tokens.comparableBaseline, 100);
assert.equal(overview.metrics.tokens.delta, -10);
});

test("groupRunFailures returns empty for missing or empty reports", () => {
assert.deepEqual(groupRunFailures(undefined), []);
assert.deepEqual(groupRunFailures([]), []);
assert.deepEqual(
groupRunFailures([
{
subject_id: "s",
report: {
scenarios: [
{
scenario_id: "clean",
runs: [{ failures: [], hard_gates: [{ id: "g", passed: true }] }],
},
],
},
},
]),
[],
);
});

test("groupRunFailures groups failures and failed gates per run", () => {
const groups = groupRunFailures([
{
subject_id: "anthropic-sonnet",
report: {
scenarios: [
{
scenario_id: "security_review",
runs: [
{
failures: [{ phase: "execute", message: "boom" }],
hard_gates: [
{ id: "no_secrets", passed: false, reason: "leaked" },
{ id: "compiles", passed: true },
],
},
{ failures: [], hard_gates: [] },
{ failures: [{ message: "" }] },
],
},
],
},
},
]);
assert.equal(groups.length, 2);
assert.deepEqual(groups[0], {
subjectId: "anthropic-sonnet",
scenarioId: "security_review",
runIndex: 0,
items: [
{ kind: "failure", phase: "execute", message: "boom" },
{ kind: "gate", gateId: "no_secrets", message: "leaked" },
],
});
assert.equal(groups[1].runIndex, 2);
assert.deepEqual(groups[1].items, [
{ kind: "failure", phase: "failure", message: "No failure message" },
]);
});
22 changes: 15 additions & 7 deletions .github/benchmark-site/execution.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,16 +92,12 @@ <h2 id="failures-title">Execution failures</h2>
<div id="detail-runtime" class="kpi-value">—</div>
<div id="workflow-runtime" class="kpi-delta">—</div>
</article>
<article class="kpi-card">
<div class="kpi-label">Reliability events</div>
<div id="detail-reliability" class="kpi-value">—</div>
<div class="kpi-delta">Blocking failures and missing reports</div>
</article>
</section>

<div class="detail-layout">
<aside class="detail-index" aria-label="Execution detail sections">
<span>On this page</span>
<a href="#detail-failures">Failures</a>
<a href="#overview">Overview</a>
<a href="#configuration">Configuration</a>
<a href="#scenarios">Scenarios and runs</a>
Expand All @@ -112,13 +108,25 @@ <h2 id="failures-title">Execution failures</h2>
<section id="overview" class="detail-section" aria-labelledby="overview-heading">
<div class="section-kicker">Execution context</div>
<h2 id="overview-heading">Overview</h2>
<div id="metadata-grid" class="metadata-grid"></div>
<details class="section-disclosure">
<summary>
<span id="overview-digest" class="section-digest">Loading…</span>
<span class="section-chevron" aria-hidden="true">⌄</span>
</summary>
<div id="metadata-grid" class="metadata-grid"></div>
</details>
</section>

<section id="configuration" class="detail-section" aria-labelledby="configuration-heading">
<div class="section-kicker">Resolved stack</div>
<h2 id="configuration-heading">Configuration</h2>
<div id="configuration-content"></div>
<details class="section-disclosure">
<summary>
<span id="configuration-digest" class="section-digest">Loading…</span>
<span class="section-chevron" aria-hidden="true">⌄</span>
</summary>
<div id="configuration-content"></div>
</details>
</section>

<section id="scenarios" class="detail-section" aria-labelledby="scenarios-heading">
Expand Down
Loading
Loading