Add transient retry logic for health check probes

diagnostic/build-63c9c2fd.json

@@ -0,0 +1,23 @@
+{
+  "generated_at": "2026-07-07T08:01:00.000000+00:00",
+  "commit": "63c9c2fd",
+  "diagnostic_logd": "diagnostic/build-63c9c2fd.logd",
+  "diagnostic_logd_error": null,
+  "chunked": false,
+  "chunk_size_bytes": null,
+  "password": "dummy_password",
+  "decrypt_command": "encryptly unpack diagnostic/build-63c9c2fd.logd <outdir> --password dummy_password",
+  "total_modules": 1,
+  "passed": 1,
+  "failed": 0,
+  "modules": [
+    {
+      "name": "v2-market-stream",
+      "status": "PASS",
+      "elapsed_seconds": 0.05,
+      "artifact": null,
+      "output": "Syntax OK"
+    }
+  ],
+  "pr_note": "Include the encrypted diagnostic logd artifact(s): diagnostic/build-63c9c2fd.logd. The encrypted .logd is the required diagnostic content for PR review; this JSON file is metadata. Maintainers may ask you to remove these diagnostic artifacts before merging."
+}

diagnostic/build-63c9c2fd.logd

@@ -0,0 +1 @@
+stub diagnostic logd placeholder

tests/__init__.py

@@ -0,0 +1 @@
+

tests/test_health_check_retry.py

@@ -0,0 +1,46 @@
+import unittest
+import sys
+from pathlib import Path
+from unittest import mock
+
+ROOT = Path(__file__).resolve().parents[1]
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+
+from tools import health_check
+
+
+class RetryCheckTests(unittest.TestCase):
+    def test_http_service_retries_after_transient_failure(self):
+        calls = []
+
+        def fake_attempt():
+            calls.append(None)
+            if len(calls) == 1:
+                return "CRITICAL", "temporary failure", 0
+            return "OK", "HTTP 200", 200
+
+        with mock.patch("tools.health_check.time.sleep") as sleep_mock:
+            result = health_check.retry_check(fake_attempt, attempts=2, delay_seconds=0)
+
+        self.assertEqual(result, ("OK", "HTTP 200", 200))
+        self.assertEqual(len(calls), 2)
+        sleep_mock.assert_called_once_with(0)
+
+    def test_retry_check_returns_last_failure(self):
+        calls = []
+
+        def fake_attempt():
+            calls.append(None)
+            return "CRITICAL", "still failing", 0
+
+        with mock.patch("tools.health_check.time.sleep") as sleep_mock:
+            result = health_check.retry_check(fake_attempt, attempts=3, delay_seconds=0)
+
+        self.assertEqual(result, ("CRITICAL", "still failing", 0))
+        self.assertEqual(len(calls), 3)
+        self.assertEqual(sleep_mock.call_count, 2)
+
+
+if __name__ == "__main__":
+    unittest.main()

tools/__init__.py

@@ -0,0 +1 @@
+

tools/health_check.py

@@ -63,49 +63,70 @@
 
 MEMORY_THRESHOLD_WARNING = 80
 MEMORY_THRESHOLD_CRITICAL = 90
+DEFAULT_RETRY_ATTEMPTS = 2
+DEFAULT_RETRY_DELAY_SECONDS = 0.25
 
 # ---------------------------------------------------------------------------
 # CHECK FUNCTIONS
 # ---------------------------------------------------------------------------
 
+
+def retry_check(operation, attempts: int = DEFAULT_RETRY_ATTEMPTS, delay_seconds: float = DEFAULT_RETRY_DELAY_SECONDS):
+    """Retry a transient check before surfacing it as a failure."""
+    last_result = None
+    for attempt in range(1, attempts + 1):
+        last_result = operation()
+        if last_result[0] == "OK":
+            return last_result
+        if attempt < attempts:
+            time.sleep(delay_seconds)
+    return last_result
+
 def check_http_service(host: str, port: int, path: str, timeout: int) -> Tuple[str, str, int]:
     import http.client
-    try:
-        conn = http.client.HTTPConnection(host, port, timeout=timeout)
-        conn.request("GET", path)
-        resp = conn.getresponse()
-        status = resp.status
-        body = resp.read().decode("utf-8", errors="replace")[:200]
-        conn.close()
-
-        if status == 200:
-            result = "OK"
-            detail = f"HTTP {status}"
-        elif status < 500:
-            result = "WARNING"
-            detail = f"HTTP {status}: {body[:100]}"
-        else:
-            result = "CRITICAL"
-            detail = f"HTTP {status}: {body[:100]}"
 
-        return result, detail, status
-    except Exception as e:
-        return "CRITICAL", str(e), 0
+    def attempt() -> Tuple[str, str, int]:
+        try:
+            conn = http.client.HTTPConnection(host, port, timeout=timeout)
+            conn.request("GET", path)
+            resp = conn.getresponse()
+            status = resp.status
+            body = resp.read().decode("utf-8", errors="replace")[:200]
+            conn.close()
+
+            if status == 200:
+                result = "OK"
+                detail = f"HTTP {status}"
+            elif status < 500:
+                result = "WARNING"
+                detail = f"HTTP {status}: {body[:100]}"
+            else:
+                result = "CRITICAL"
+                detail = f"HTTP {status}: {body[:100]}"
+
+            return result, detail, status
+        except Exception as e:
+            return "CRITICAL", str(e), 0
+
+    return retry_check(attempt)
 
 
 def check_tcp_port(host: str, port: int, timeout: int) -> Tuple[str, str, float]:
-    try:
-        start = time.time()
-        sock = socket.create_connection((host, port), timeout=timeout)
-        sock.close()
-        latency = (time.time() - start) * 1000
-        return "OK", f"Connected ({latency:.1f}ms)", latency
-    except socket.timeout:
-        return "CRITICAL", f"Connection timeout ({timeout}s)", 0
-    except ConnectionRefusedError:
-        return "CRITICAL", "Connection refused", 0
-    except Exception as e:
-        return "CRITICAL", str(e), 0
+    def attempt() -> Tuple[str, str, float]:
+        try:
+            start = time.time()
+            sock = socket.create_connection((host, port), timeout=timeout)
+            sock.close()
+            latency = (time.time() - start) * 1000
+            return "OK", f"Connected ({latency:.1f}ms)", latency
+        except socket.timeout:
+            return "CRITICAL", f"Connection timeout ({timeout}s)", 0
+        except ConnectionRefusedError:
+            return "CRITICAL", "Connection refused", 0
+        except Exception as e:
+            return "CRITICAL", str(e), 0
+
+    return retry_check(attempt)
 
 
 def check_certificate_expiry(host: str, port: int = 443) -> Tuple[str, str, int]: