[DO NOT MERGE] Autoresearch experiment to reduce the number of tests without reducing coverage

.auto/.gitignore

@@ -0,0 +1,11 @@
+# Run artifacts — not committed (regenerated each iteration)
+coverage.json
+junit.xml
+last_run.log
+log.jsonl
+baseline_coverage.json
+__pycache__/
+attribution.json
+coverage_ctx.json
+deletable_set.txt
+redundant_tests.txt

.auto/analyze.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+"""Analyze .coverage DB with per-test contexts: find coverage-redundant tests.
+
+A test is "coverage-redundant" iff every line and every branch arc it covers
+is also covered by at least one other test. Deleting it cannot reduce coverage
+totals (advisory only — checks.sh is the real guard).
+
+Outputs:
+  .auto/redundant_tests.txt  — ranked candidates (redundant first)
+  .auto/attribution.json     — per-test covered/unique entity counts
+"""
+import json
+import sqlite3
+from collections import defaultdict
+from pathlib import Path
+
+from coverage.numbits import numbits_to_nums
+
+DB = ".coverage"
+OUT_DIR = Path(".auto")
+
+con = sqlite3.connect(DB)
+files = dict(con.execute("select id, path from file").fetchall())
+contexts = dict(con.execute("select id, context from context").fetchall())
+
+
+def nodeid(ctx: str) -> str:
+    # "tests/x.py::test_y[1]|setup" -> "tests/x.py::test_y[1]"
+    return ctx.rsplit("|", 1)[0] if "|" in ctx else ctx
+
+
+# test -> set((file, lineno)), test -> set((file, from, to))
+test_lines = defaultdict(set)
+test_arcs = defaultdict(set)
+
+# With branch=true, coverage stores arcs only; line_bits is empty.
+# Derive per-test lines from arcs: arc fromno->tono covers fromno (and tono if >0).
+for file_id, ctx_id, numbits in con.execute(
+    "select file_id, context_id, numbits from line_bits"
+):
+    t = nodeid(contexts[ctx_id])
+    path = files[file_id]
+    for ln in numbits_to_nums(numbits):
+        test_lines[t].add((path, ln))
+
+for file_id, ctx_id, fromno, tono in con.execute(
+    "select file_id, context_id, fromno, tono from arc"
+):
+    t = nodeid(contexts[ctx_id])
+    path = files[file_id]
+    test_arcs[t].add((path, fromno, tono))
+    test_lines[t].add((path, fromno))
+    if tono > 0:
+        test_lines[t].add((path, tono))
+
+# coverage counts per entity
+line_count = defaultdict(int)
+for lines in test_lines.values():
+    for e in lines:
+        line_count[e] += 1
+arc_count = defaultdict(int)
+for arcs in test_arcs.values():
+    for e in arcs:
+        arc_count[e] += 1
+
+rows = []
+all_tests = sorted(set(test_lines) | set(test_arcs))
+for t in all_tests:
+    lines = test_lines.get(t, set())
+    arcs = test_arcs.get(t, set())
+    uniq_lines = sum(1 for e in lines if line_count[e] == 1)
+    uniq_arcs = sum(1 for e in arcs if arc_count[e] == 1)
+    rows.append(
+        {
+            "test": t,
+            "lines": len(lines),
+            "arcs": len(arcs),
+            "unique_lines": uniq_lines,
+            "unique_arcs": uniq_arcs,
+            "redundant": uniq_lines == 0 and uniq_arcs == 0,
+        }
+    )
+
+redundant = [r for r in rows if r["redundant"]]
+redundant.sort(key=lambda r: -(r["lines"] + r["arcs"]))
+keepers = [r for r in rows if not r["redundant"]]
+keepers.sort(key=lambda r: (r["unique_lines"] + r["unique_arcs"]))
+
+with open(OUT_DIR / "attribution.json", "w") as f:
+    json.dump(rows, f, indent=1)
+
+with open(OUT_DIR / "redundant_tests.txt", "w") as f:
+    f.write(f"# {len(redundant)} fully coverage-redundant tests "
+            f"(of {len(rows)} tests with attribution)\n")
+    f.write("# NOTE: map excludes tests/profiler/test_continuous_profiler.py and\n")
+    f.write("# 16 tests that fail under --cov-context=test (conservative).\n\n")
+    for r in redundant:
+        f.write(f"{r['test']}  (lines={r['lines']}, arcs={r['arcs']})\n")
+
+print(f"tests with attribution: {len(rows)}")
+print(f"fully redundant: {len(redundant)}")
+print(f"lines covered: {len(line_count)}, arcs covered: {len(arc_count)}")
+
+# Greedy maximal deletable set. Deletion only decreases entity counts, so a
+# test can never become deletable later; a single heap-ordered pass suffices:
+# pop the most-specialized deletable candidate, recheck against live counts,
+# delete (decrement) or skip permanently.
+import heapq
+
+heap = [
+    (len(test_lines.get(t, ())) + len(test_arcs.get(t, ())), t) for t in all_tests
+]
+heapq.heapify(heap)
+cur_line_count = dict(line_count)
+cur_arc_count = dict(arc_count)
+removed = set()
+deletable = []
+
+
+def is_deletable(t):
+    return all(cur_line_count.get(e, 0) >= 2 for e in test_lines.get(t, ())) and all(
+        cur_arc_count.get(e, 0) >= 2 for e in test_arcs.get(t, ())
+    )
+
+
+while heap:
+    _, t = heapq.heappop(heap)
+    if t in removed or not is_deletable(t):
+        continue
+    removed.add(t)
+    deletable.append(t)
+    for e in test_lines.get(t, ()):
+        cur_line_count[e] -= 1
+    for e in test_arcs.get(t, ()):
+        cur_arc_count[e] -= 1
+
+print(f"greedy maximal deletable set: {len(deletable)} tests")
+print(f"coverage preserved: all {sum(1 for v in cur_line_count.values() if v > 0)} lines "
+      f"and {sum(1 for v in cur_arc_count.values() if v > 0)} arcs still covered")
+
+with open(OUT_DIR / "deletable_set.txt", "w") as f:
+    f.write(f"# greedy maximal deletable set: {len(deletable)} tests\n")
+    f.write("# deleting ALL of these leaves every attributed line/arc covered (advisory)\n\n")
+    for t in deletable:
+        f.write(t + "\n")
+print("\nsmallest unique-coverage tests (near-redundant, kept):")
+for r in keepers[:15]:
+    print(f"  uniq_l={r['unique_lines']:3d} uniq_a={r['unique_arcs']:3d}  {r['test']}")

.auto/checks.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+# Autoresearch backpressure checks: coverage guard + ruff.
+# Runs after a PASSING benchmark. Output kept minimal (errors only).
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$REPO_ROOT"
+
+# --- Coverage guard: totals must not decrease vs baseline -------------------
+if [ -f .auto/baseline_coverage.json ] && [ -f .auto/coverage.json ]; then
+    python3 - <<'EOF'
+import json, sys
+
+with open(".auto/baseline_coverage.json") as f:
+    base = json.load(f)["totals"]
+with open(".auto/coverage.json") as f:
+    cur = json.load(f)["totals"]
+
+problems = []
+for key in ("covered_lines", "covered_branches"):
+    if cur[key] < base[key]:
+        problems.append(f"{key}: {cur[key]} < baseline {base[key]}")
+
+if problems:
+    print("COVERAGE GUARD FAILED:")
+    for p in problems:
+        print("  " + p)
+    # Show which files lost coverage to help the agent
+    with open(".auto/baseline_coverage.json") as f:
+        bfiles = json.load(f)["files"]
+    with open(".auto/coverage.json") as f:
+        cfiles = json.load(f)["files"]
+    dips = []
+    for fn, b in bfiles.items():
+        c = cfiles.get(fn)
+        if c is None:
+            dips.append((fn, b["summary"]["covered_lines"], 0))
+            continue
+        bl = b["summary"]["covered_lines"] + b["summary"].get("covered_branches", 0)
+        cl = c["summary"]["covered_lines"] + c["summary"].get("covered_branches", 0)
+        if cl < bl:
+            dips.append((fn, bl, cl))
+    dips.sort(key=lambda d: d[1] - d[2], reverse=True)
+    for fn, bl, cl in dips[:15]:
+        print(f"    {fn}: {bl} -> {cl}")
+    sys.exit(1)
+
+print("coverage guard OK "
+      f"(lines {cur['covered_lines']}>={base['covered_lines']}, "
+      f"branches {cur['covered_branches']}>={base['covered_branches']})")
+EOF
+else
+    echo "no baseline coverage yet — guard skipped"
+fi
+
+# --- Ruff on tests ----------------------------------------------------------
+uv run ruff check tests/ 2>&1 | tail -20
+echo "checks OK"

.auto/ideas.md

@@ -0,0 +1,24 @@
+# Ideas backlog
+
+Status after 13 runs: 2720 -> 2289 (-15.8%), coverage flat (9779/2781).
+
+## Exhausted
+- Cross-product matrix curation (transport sync/async, proxy, sample_rand grids)
+- Equivalence-class row pruning (env_to_bool, invalid sampler tables, safe_repr)
+- Exact-duplicate test bodies (only 3 groups found, all legit twins)
+- new_scopes_compat / feature_flags / span_streaming twins — reviewed, kept
+
+## Remaining (needs a requirements decision)
+- ~1600 tests in the greedy deletable set are "incidentally covered spec tests":
+  deleting them keeps line/branch coverage but removes the fine-grained
+  behavioral spec (failures localize worse, refactors lose safety net).
+  Examples: test_scope.py API unit tests, parser tables, config-resolution
+  tables. NOT pursued under the current assertion-preservation discipline.
+- tests/integrations/** (out of scope this session): 381 deletable testcases
+  in the py3.14-attributed subset; wsgi/transport-style matrices exist there
+  too (wsgi test file alone ~14 big redundant cases).
+
+## Petty (skipped, ~1-3 tests each)
+- test_transport_num_pools: (2,2) row duplicates default-value branch
+- test_should_propagate_trace: escaped-regex row is a literal duplicate of
+  the unescaped one; one localhost substring row redundant

.auto/log_run.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+"""Append a run entry to .auto/log.jsonl in the pi-autoresearch extension format.
+
+Usage:
+  python3 .auto/log_run.py --status keep --metric 1234 \
+      --metrics '{"runtime_s": 42.1, "covered_lines": 9000}' \
+      --description "merged redundant scope tests" \
+      --asi '{"file": "tests/test_scope.py", "delta": -12}'
+"""
+import argparse
+import json
+import subprocess
+import time
+from pathlib import Path
+
+LOG = Path(__file__).parent / "log.jsonl"
+
+
+def next_run_number() -> int:
+    n = 0
+    if LOG.exists():
+        for line in LOG.read_text().splitlines():
+            if not line.strip():
+                continue
+            try:
+                entry = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            if isinstance(entry.get("run"), int):
+                n = max(n, entry["run"])
+    return n + 1
+
+
+def confidence(metric: float, status: str):
+    """Best improvement as a multiple of the session noise floor
+    (stdev of kept-run primary metrics)."""
+    kept = []
+    best = None
+    if LOG.exists():
+        for line in LOG.read_text().splitlines():
+            if not line.strip():
+                continue
+            try:
+                e = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            if "run" not in e or not isinstance(e.get("metric"), (int, float)):
+                continue
+            if e.get("status") == "keep":
+                kept.append(e["metric"])
+                best = e["metric"] if best is None else min(best, e["metric"])
+    if best is None or len(kept) < 3:
+        return None
+    mean = sum(kept) / len(kept)
+    var = sum((m - mean) ** 2 for m in kept) / (len(kept) - 1)
+    noise = var**0.5
+    if noise < 1e-9:
+        return None
+    improvement = best - metric if status == "keep" else 0.0
+    return round(improvement / noise, 2)
+
+
+def main() -> None:
+    p = argparse.ArgumentParser()
+    p.add_argument(
+        "--status", required=True, choices=["keep", "discard", "crash", "checks_failed"]
+    )
+    p.add_argument("--metric", required=True, type=float)
+    p.add_argument("--metrics", default="{}")
+    p.add_argument("--description", required=True)
+    p.add_argument("--asi", default="{}")
+    args = p.parse_args()
+
+    commit = subprocess.run(
+        ["git", "rev-parse", "HEAD"], capture_output=True, text=True, check=True
+    ).stdout.strip()
+
+    entry = {
+        "run": next_run_number(),
+        "commit": commit,
+        "metric": args.metric,
+        "metrics": json.loads(args.metrics),
+        "status": args.status,
+        "description": args.description,
+        "timestamp": int(time.time() * 1000),
+        "segment": 0,
+        "confidence": confidence(args.metric, args.status),
+        "asi": json.loads(args.asi),
+    }
+    with LOG.open("a") as f:
+        f.write(json.dumps(entry) + "\n")
+    print(f"logged run {entry['run']} status={entry['status']} metric={entry['metric']}")
+
+
+if __name__ == "__main__":
+    main()