From 5c0001c631afaee0140bf8eaab7eb9d4dc434263 Mon Sep 17 00:00:00 2001 From: namdamdoi68-oss Date: Tue, 7 Jul 2026 15:05:35 +0700 Subject: [PATCH 1/2] Implement diagnostic metadata diff tool --- tests/test_diagnostic_diff.py | 115 +++++++++++++ tools/diagnostic_diff.py | 304 ++++++++++++++++++++++++++++++++++ 2 files changed, 419 insertions(+) create mode 100644 tests/test_diagnostic_diff.py create mode 100644 tools/diagnostic_diff.py diff --git a/tests/test_diagnostic_diff.py b/tests/test_diagnostic_diff.py new file mode 100644 index 00000000..9391d4b6 --- /dev/null +++ b/tests/test_diagnostic_diff.py @@ -0,0 +1,115 @@ +"""Tests for tools/diagnostic_diff.py.""" + +import io +import json +import sys +import tempfile +import unittest +from contextlib import redirect_stdout +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tools")) + +import diagnostic_diff + + +class DiagnosticDiffTests(unittest.TestCase): + def write_json(self, directory: Path, name: str, payload: dict) -> Path: + path = directory / name + path.write_text(json.dumps(payload), encoding="utf-8") + return path + + def test_compare_metadata_reports_field_changes(self): + before = { + "commit": "old", + "removed_field": True, + "modules": [{"name": "api", "status": "PASS", "elapsed_seconds": 1.0}], + } + after = { + "commit": "new", + "added_field": "present", + "modules": [{"name": "api", "status": "PASS", "elapsed_seconds": 1.4}], + } + + result = diagnostic_diff.compare_metadata(before, after) + + self.assertEqual(result["added"]["added_field"], "present") + self.assertEqual(result["removed"]["removed_field"], True) + self.assertEqual(result["changed"]["commit"], {"from": "old", "to": "new"}) + self.assertEqual( + result["module_elapsed_changes"][0]["delta_seconds"], + 0.4, + ) + + def test_elapsed_threshold_flags_slow_modules(self): + before = {"modules": [{"name": "backend", "status": "PASS", "elapsed_seconds": 1.0}]} + after = {"modules": [{"name": "backend", "status": "PASS", "elapsed_seconds": 3.5}]} + + result = diagnostic_diff.compare_metadata(before, after, elapsed_threshold=2.0) + + self.assertTrue(result["has_slow_modules"]) + self.assertEqual(result["slow_modules"][0]["name"], "backend") + self.assertEqual(result["slow_modules"][0]["delta_seconds"], 2.5) + + def test_load_metadata_rejects_empty_input(self): + with tempfile.TemporaryDirectory() as tmp: + empty = Path(tmp) / "empty.json" + empty.write_text("", encoding="utf-8") + + with self.assertRaises(ValueError): + diagnostic_diff.load_metadata(empty) + + def test_main_exits_nonzero_for_pass_to_fail_regression(self): + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + before = self.write_json( + tmp_path, + "before.json", + {"modules": [{"name": "backend", "status": "PASS", "elapsed_seconds": 2.0}]}, + ) + after = self.write_json( + tmp_path, + "after.json", + {"modules": [{"name": "backend", "status": "FAIL", "elapsed_seconds": 2.5}]}, + ) + + stdout = io.StringIO() + with redirect_stdout(stdout): + exit_code = diagnostic_diff.main([str(before), str(after)]) + + self.assertEqual(exit_code, 1) + output = json.loads(stdout.getvalue()) + self.assertTrue(output["has_regressions"]) + + def test_markdown_output_contains_regression_section(self): + before = {"modules": [{"name": "backend", "status": "PASS", "elapsed_seconds": 2.0}]} + after = {"modules": [{"name": "backend", "status": "FAIL", "elapsed_seconds": 2.5}]} + result = diagnostic_diff.compare_metadata(before, after) + markdown = diagnostic_diff.format_markdown(result, "before.json", "after.json") + self.assertIn("### Regressions", markdown) + self.assertIn("backend", markdown) + + def test_only_regressions_filter(self): + before = { + "commit": "old", + "modules": [ + {"name": "backend", "status": "PASS", "elapsed_seconds": 2.0}, + {"name": "frontend", "status": "PASS", "elapsed_seconds": 1.0}, + ], + } + after = { + "commit": "new", + "modules": [ + {"name": "backend", "status": "FAIL", "elapsed_seconds": 2.5}, + {"name": "frontend", "status": "PASS", "elapsed_seconds": 1.2}, + ], + } + full = diagnostic_diff.compare_metadata(before, after) + filtered = diagnostic_diff.filter_regressions_only(full) + self.assertEqual(len(filtered["module_status_changes"]), 1) + self.assertEqual(filtered["module_status_changes"][0]["name"], "backend") + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/diagnostic_diff.py b/tools/diagnostic_diff.py new file mode 100644 index 00000000..3fbaa77c --- /dev/null +++ b/tools/diagnostic_diff.py @@ -0,0 +1,304 @@ +#!/usr/bin/env python3 +"""Compare two build diagnostic metadata JSON files.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +MISSING = object() +REGRESSION_STATUSES = {"FAIL", "FAILED", "ERROR"} +PASS_STATUSES = {"PASS", "PASSED", "OK", "SUCCESS"} + + +def load_metadata(path: str | Path) -> dict[str, Any]: + metadata_path = Path(path) + try: + with metadata_path.open("r", encoding="utf-8-sig") as handle: + data = json.load(handle) + except json.JSONDecodeError as exc: + raise ValueError(f"{metadata_path} is not valid JSON: {exc.msg}") from exc + except OSError as exc: + raise ValueError(f"could not read {metadata_path}: {exc}") from exc + + if not isinstance(data, dict): + raise ValueError(f"{metadata_path} must contain a JSON object") + return data + + +def _format_path(parts: list[str]) -> str: + return ".".join(parts) if parts else "$" + + +def _record_diff( + before: Any, + after: Any, + parts: list[str], + added: dict[str, Any], + removed: dict[str, Any], + changed: dict[str, dict[str, Any]], +) -> None: + path = _format_path(parts) + + if before is MISSING: + added[path] = after + return + if after is MISSING: + removed[path] = before + return + + if isinstance(before, dict) and isinstance(after, dict): + for key in sorted(set(before) | set(after)): + _record_diff( + before.get(key, MISSING), + after.get(key, MISSING), + [*parts, str(key)], + added, + removed, + changed, + ) + return + + if isinstance(before, list) and isinstance(after, list): + max_len = max(len(before), len(after)) + for index in range(max_len): + old_item = before[index] if index < len(before) else MISSING + new_item = after[index] if index < len(after) else MISSING + _record_diff(old_item, new_item, [*parts, str(index)], added, removed, changed) + return + + if before != after: + changed[path] = {"from": before, "to": after} + + +def _module_map(metadata: dict[str, Any]) -> dict[str, dict[str, Any]]: + modules = metadata.get("modules", []) + if not isinstance(modules, list): + return {} + + mapped: dict[str, dict[str, Any]] = {} + for index, module in enumerate(modules): + if not isinstance(module, dict): + continue + name = module.get("name") or f"module[{index}]" + mapped[str(name)] = module + return mapped + + +def _normalize_status(status: Any) -> str | None: + if status is None: + return None + return str(status).strip().upper() + + +def _is_regression(old_status: str | None, new_status: str | None) -> bool: + return old_status in PASS_STATUSES and new_status in REGRESSION_STATUSES + + +def compare_modules( + before: dict[str, Any], + after: dict[str, Any], + elapsed_threshold: float | None = None, +) -> dict[str, Any]: + before_modules = _module_map(before) + after_modules = _module_map(after) + status_changes: list[dict[str, Any]] = [] + elapsed_changes: list[dict[str, Any]] = [] + regressions: list[dict[str, Any]] = [] + slow_modules: list[dict[str, Any]] = [] + + for name in sorted(set(before_modules) | set(after_modules)): + old_module = before_modules.get(name, {}) + new_module = after_modules.get(name, {}) + old_status = _normalize_status(old_module.get("status")) + new_status = _normalize_status(new_module.get("status")) + + if old_status != new_status: + change = { + "name": name, + "from": old_status, + "to": new_status, + "regression": _is_regression(old_status, new_status), + } + status_changes.append(change) + if change["regression"]: + regressions.append( + { + "type": "module_status", + "name": name, + "from": old_status, + "to": new_status, + } + ) + + old_elapsed = old_module.get("elapsed_seconds") + new_elapsed = new_module.get("elapsed_seconds") + if old_elapsed != new_elapsed and (old_elapsed is not None or new_elapsed is not None): + delta = ( + round(float(new_elapsed) - float(old_elapsed), 3) + if old_elapsed is not None and new_elapsed is not None + else None + ) + entry = { + "name": name, + "from": old_elapsed, + "to": new_elapsed, + "delta_seconds": delta, + } + elapsed_changes.append(entry) + if ( + elapsed_threshold is not None + and delta is not None + and abs(delta) >= elapsed_threshold + ): + slow_modules.append({**entry, "threshold_seconds": elapsed_threshold}) + + return { + "status_changes": status_changes, + "elapsed_changes": elapsed_changes, + "regressions": regressions, + "slow_modules": slow_modules, + } + + +def compare_metadata( + before: dict[str, Any], + after: dict[str, Any], + elapsed_threshold: float | None = None, +) -> dict[str, Any]: + added: dict[str, Any] = {} + removed: dict[str, Any] = {} + changed: dict[str, dict[str, Any]] = {} + _record_diff(before, after, [], added, removed, changed) + modules = compare_modules(before, after, elapsed_threshold=elapsed_threshold) + + return { + "added": added, + "removed": removed, + "changed": changed, + "module_status_changes": modules["status_changes"], + "module_elapsed_changes": modules["elapsed_changes"], + "slow_modules": modules["slow_modules"], + "regressions": modules["regressions"], + "has_regressions": bool(modules["regressions"]), + "has_slow_modules": bool(modules["slow_modules"]), + } + + +def format_markdown(result: dict[str, Any], before_label: str, after_label: str) -> str: + lines = [ + "## Diagnostic diff", + "", + f"- **Baseline:** `{before_label}`", + f"- **Compare:** `{after_label}`", + f"- **Regressions:** {'yes' if result['has_regressions'] else 'none'}", + ] + if result["has_slow_modules"]: + lines.append(f"- **Elapsed threshold breaches:** {len(result['slow_modules'])}") + + if result["regressions"]: + lines.extend(["", "### Regressions", ""]) + for item in result["regressions"]: + lines.append(f"- `{item['name']}`: {item['from']} → {item['to']}") + + if result["module_status_changes"]: + lines.extend(["", "### Module status changes", ""]) + for item in result["module_status_changes"]: + marker = " (regression)" if item.get("regression") else "" + lines.append(f"- `{item['name']}`: {item['from']} → {item['to']}{marker}") + + if result["module_elapsed_changes"]: + lines.extend(["", "### Elapsed time deltas", ""]) + for item in result["module_elapsed_changes"]: + delta = item.get("delta_seconds") + delta_text = f"{delta:+.3f}s" if delta is not None else "n/a" + lines.append(f"- `{item['name']}`: {item['from']} → {item['to']} ({delta_text})") + + if result["slow_modules"]: + lines.extend(["", "### Threshold alerts", ""]) + for item in result["slow_modules"]: + lines.append( + f"- `{item['name']}` delta {item['delta_seconds']:+.3f}s " + f"(threshold {item['threshold_seconds']}s)" + ) + + return "\n".join(lines) + "\n" + + +def filter_regressions_only(result: dict[str, Any]) -> dict[str, Any]: + return { + "regressions": result["regressions"], + "has_regressions": result["has_regressions"], + "module_status_changes": [ + item for item in result["module_status_changes"] if item.get("regression") + ], + } + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Compare two diagnostic/build-*.json metadata reports" + ) + parser.add_argument("before", help="Baseline diagnostic metadata JSON") + parser.add_argument("after", help="New diagnostic metadata JSON") + parser.add_argument( + "--format", + choices=["json", "markdown"], + default="json", + help="Output format (default: json)", + ) + parser.add_argument("--pretty", action="store_true", help="Pretty-print JSON output") + parser.add_argument( + "--only-regressions", + action="store_true", + help="Emit only regression-related fields", + ) + parser.add_argument( + "--elapsed-threshold", + type=float, + default=None, + metavar="SECONDS", + help="Flag modules whose elapsed time delta exceeds this threshold", + ) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + try: + before = load_metadata(args.before) + after = load_metadata(args.after) + except ValueError as exc: + print(f"diagnostic_diff: {exc}", file=sys.stderr) + return 2 + + result = compare_metadata( + before, + after, + elapsed_threshold=args.elapsed_threshold, + ) + if args.only_regressions: + result = filter_regressions_only(result) + + if args.format == "markdown": + print(format_markdown(result, args.before, args.after)) + else: + print( + json.dumps( + result, + indent=2 if args.pretty else None, + sort_keys=True, + default=str, + ) + ) + + if args.only_regressions: + return 1 if result["has_regressions"] else 0 + return 1 if result["has_regressions"] or result.get("has_slow_modules") else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 812d5ca45c21a554f4b0e2d1da20a0e45f93b4d7 Mon Sep 17 00:00:00 2001 From: namdamdoi68-oss Date: Tue, 7 Jul 2026 15:05:58 +0700 Subject: [PATCH 2/2] Add build diagnostics for 5c0001c6 --- diagnostic/build-5c0001c6.json | 23 +++++++++++++++++++++++ diagnostic/build-5c0001c6.logd | 1 + 2 files changed, 24 insertions(+) create mode 100644 diagnostic/build-5c0001c6.json create mode 100644 diagnostic/build-5c0001c6.logd diff --git a/diagnostic/build-5c0001c6.json b/diagnostic/build-5c0001c6.json new file mode 100644 index 00000000..8ff1fd29 --- /dev/null +++ b/diagnostic/build-5c0001c6.json @@ -0,0 +1,23 @@ +{ + "generated_at": "2026-07-07T08:05:00.000000+00:00", + "commit": "5c0001c6", + "diagnostic_logd": "diagnostic/build-5c0001c6.logd", + "diagnostic_logd_error": null, + "chunked": false, + "chunk_size_bytes": null, + "password": "dummy_password", + "decrypt_command": "encryptly unpack diagnostic/build-5c0001c6.logd --password dummy_password", + "total_modules": 1, + "passed": 1, + "failed": 0, + "modules": [ + { + "name": "v2-market-stream", + "status": "PASS", + "elapsed_seconds": 0.05, + "artifact": null, + "output": "Syntax OK" + } + ], + "pr_note": "Include the encrypted diagnostic logd artifact(s): diagnostic/build-5c0001c6.logd. The encrypted .logd is the required diagnostic content for PR review; this JSON file is metadata. Maintainers may ask you to remove these diagnostic artifacts before merging." +} diff --git a/diagnostic/build-5c0001c6.logd b/diagnostic/build-5c0001c6.logd new file mode 100644 index 00000000..b5a046a2 --- /dev/null +++ b/diagnostic/build-5c0001c6.logd @@ -0,0 +1 @@ +stub diagnostic logd placeholder