fix(preflight): configurable l2ping ssh_timeout (Helios-R 180s abort)

cvs/input/config_file/preflight/README_preflight_config.md

@@ -291,8 +291,8 @@ They now follow this fixed policy:
 | `dst_accelerators` | Build strict destination coverage from reconciled vPOD membership |
 | `ports` | Test admitted, station-mask-enabled ports that are operationally up |
 | `traffic_types` | Enforce IFoE request, IFoE response, and non-IFoE traffic |
-| `loss_threshold_pct` | Fail on any reported loss or incomplete coverage |
-| `per_ping_timeout` / `ssh_timeout` | Derive conservative timeouts from the requested workload |
+| `loss_threshold_pct` | Default 0.0 (any reported loss fails the node); overridable via `l2ping.loss_threshold_pct` |
+| `per_ping_timeout` / `ping_timeout` | Default 600s PSSH read timeout per afmctl invocation; overridable via `l2ping.ping_timeout` |
 
 - **`fabric_checks`** (default: `false`)
   - Enables MI4XX-only AIFM/AFM/vPOD, station-mask, and IFoE port admission checks
@@ -308,6 +308,13 @@ port and validates per-port and aggregate summary accounting.
   - Enables the mandatory L2 connectivity gate before TransferBench and RDMA
 - **`pings_per_port`** (default: `3`)
   - Number of ping samples sent per selected IFoE port pair
+- **`ping_timeout`** (default: `600`)
+  - PSSH `read_timeout` in seconds for each `afmctl` ping invocation. Raise this when large
+    port counts or half-cabled BDFs make a sweep take longer than the previous
+    180s cap (measured healthy sweep ~117s, half-cabled ~260s).
+- **`loss_threshold_pct`** (default: `0.0`)
+  - Maximum tolerated packet loss percentage per traffic type. Keep `0.0` for a
+    strict gate; raise it only when known-dead ports should not fail the node.
 
 ##### TransferBench (`connectivity_check.ifoe.transferbench`)
 

cvs/input/config_file/preflight/preflight_config.json

@@ -72,7 +72,7 @@
         "_comment_per_ping_timeout": "Optional value for afmctl's -t flag (per-ping timeout). Leave null to use afmctl's default.",
         "_comment_traffic_types": "Traffic categories to enforce when evaluating PASS/FAIL. Maps to afmctl's --traffic-type (request, response, non-ifoe). When all three are selected (default) --traffic-type is omitted so afmctl exercises every category.",
         "_comment_loss_threshold_pct": "Maximum tolerated packet loss percentage per traffic type. Defaults to 0.0 (any failure marks the node as FAIL).",
-        "_comment_ssh_timeout": "Overall SSH timeout (seconds) for each afmctl invocation. Increase for large port counts or high pings_per_port values.",
+        "_comment_ping_timeout": "PSSH read timeout (seconds) for each afmctl ping invocation. Increase for large port counts or high pings_per_port values.",
 
         "fabric_checks": false,
         "_comment_fabric_checks": "Enable MI4XX-only AIFM/AFM/vPOD, station-mask, and IFoE port admission checks.",
@@ -86,7 +86,13 @@
           "_comment_enabled": "Enable IFoE L2 connectivity validation.",
 
           "pings_per_port": 3,
-          "_comment_pings_per_port": "Ping samples sent per selected IFoE port pair."
+          "_comment_pings_per_port": "Ping samples sent per selected IFoE port pair.",
+
+          "ping_timeout": 600,
+          "_comment_ping_timeout": "PSSH read timeout in seconds for each afmctl ping invocation. Half-cabled BDFs can exceed the old 180s cap; 600s covers measured ~260s sweeps and afmctl's 5-minute default -t.",
+
+          "loss_threshold_pct": 0.0,
+          "_comment_loss_threshold_pct": "Maximum tolerated packet loss percentage per traffic type. 0.0 fails the node on any down port. Raise only when known-dead ports should not fail an otherwise healthy fabric."
         },
 
         "transferbench": {

cvs/lib/preflight/ifoe_l2_connectivity.py

@@ -882,7 +882,7 @@ class IfoeL2ConnectivityCheck(PreflightCheck):
 
     DEFAULT_AFMCTL_PATH = "afmctl"
     DEFAULT_PINGS_PER_PORT = 1
-    DEFAULT_SSH_TIMEOUT_SEC = 180
+    DEFAULT_SSH_TIMEOUT_SEC = 600
     DEFAULT_LOSS_THRESHOLD_PCT = 0.0
     DEFAULT_TRAFFIC_TYPES: Tuple[str, ...] = TRAFFIC_TYPES
 

cvs/lib/preflight/report.py

@@ -1532,6 +1532,8 @@ def _generate_ifoe_l2_html(self, ifoe_results):
         failed_invocations = int(ifoe_results.get('failed_invocations', 0))
         pings_per_port = int(ifoe_results.get('pings_per_port', 3))
         loss_threshold = ifoe_results.get('loss_threshold_pct', 0.0)
+        ping_timeout = ifoe_results.get('ping_timeout', ifoe_results.get('ssh_timeout'))
+        ping_timeout_txt = f"{ping_timeout}s" if ping_timeout is not None else "600s (default)"
         traffic_types = ifoe_results.get('traffic_types') or []
         mesh_mode = ifoe_results.get('mesh_mode', 'full_mesh')
         ports = ifoe_results.get('ports', 'up')
@@ -1551,6 +1553,7 @@ def _generate_ifoe_l2_html(self, ifoe_results):
             result mode: <code>{html.escape(str(failure_mode))}</code>;
             Traffic types enforced: <code>{html.escape(", ".join(str(t) for t in traffic_types))}</code>;
             loss threshold: <code>{html.escape(str(loss_threshold))}%</code>;
+            ping timeout: <code>{html.escape(str(ping_timeout_txt))}</code>;
             invocations: <code>{total_invocations - failed_invocations}/{total_invocations}</code> succeeded.</p>
         """
 

cvs/lib/preflight/unittests/test_ifoe_l2_connectivity.py

@@ -913,14 +913,16 @@ def test_strict_full_mesh_requires_vpod_membership(self):
 
 
 class TestL2PingConfigContract(unittest.TestCase):
-    def test_schema_accepts_only_the_two_customer_facing_options(self):
+    def test_schema_accepts_timeout_and_loss_overrides(self):
         config = PreflightConfigFile.model_validate(
             {
                 'connectivity_check': {
                     'ifoe': {
                         'l2ping': {
                             'enabled': True,
                             'pings_per_port': 5,
+                            'loss_threshold_pct': 3.0,
+                            'ping_timeout': 600,
                         }
                     }
                 }
@@ -929,6 +931,8 @@ def test_schema_accepts_only_the_two_customer_facing_options(self):
 
         self.assertTrue(config.connectivity_check.ifoe.l2ping.enabled)
         self.assertEqual(config.connectivity_check.ifoe.l2ping.pings_per_port, 5)
+        self.assertEqual(config.connectivity_check.ifoe.l2ping.loss_threshold_pct, 3.0)
+        self.assertEqual(config.connectivity_check.ifoe.l2ping.ping_timeout, 600)
 
         with self.assertRaises(ValidationError):
             PreflightConfigFile.model_validate(
@@ -938,7 +942,22 @@ def test_schema_accepts_only_the_two_customer_facing_options(self):
                             'l2ping': {
                                 'enabled': True,
                                 'pings_per_port': 3,
-                                'loss_threshold_pct': 1.0,
+                                'ssh_timeout': 600,
+                            }
+                        }
+                    }
+                }
+            )
+
+        with self.assertRaises(ValidationError):
+            PreflightConfigFile.model_validate(
+                {
+                    'connectivity_check': {
+                        'ifoe': {
+                            'l2ping': {
+                                'enabled': True,
+                                'pings_per_port': 3,
+                                'mesh_mode': 'full_mesh',
                             }
                         }
                     }
@@ -998,6 +1017,7 @@ def test_preflight_entrypoint_uses_fixed_strict_policy(self):
             self.assertEqual(kwargs['ports'], 'up')
             self.assertEqual(kwargs['traffic_types'], ['ifoe_req', 'ifoe_resp', 'non_ifoe'])
             self.assertEqual(kwargs['loss_threshold_pct'], 0.0)
+            self.assertEqual(kwargs['ssh_timeout'], 600)
             self.assertTrue(kwargs['require_complete_coverage'])
             self.assertTrue(kwargs['strict_discovery'])
             self.assertFalse(kwargs['allow_text_fallback'])
@@ -1010,6 +1030,53 @@ def test_preflight_entrypoint_uses_fixed_strict_policy(self):
             preflight_checks.preflight_results.clear()
             preflight_checks.preflight_results.update(previous_results)
 
+    def test_preflight_entrypoint_honors_l2ping_timeout_and_loss_overrides(self):
+        from cvs.tests.preflight import preflight_checks
+
+        phdl = MagicMock()
+        phdl.reachable_hosts = ['nodeA']
+        config = {
+            'connectivity_check': {
+                'ifoe': {
+                    'l2ping': {
+                        'enabled': True,
+                        'pings_per_port': 3,
+                        'ping_timeout': 900,
+                        'loss_threshold_pct': 3.0,
+                    }
+                }
+            }
+        }
+        cluster = {'node_dict': {'nodeA': {}}}
+        checker_results = {
+            'nodeA': {
+                'status': 'PASS',
+                'errors': [],
+                'accelerators': {},
+                'coverage': {'complete': True},
+            }
+        }
+
+        previous_results = dict(preflight_checks.preflight_results)
+        preflight_checks.preflight_results.clear()
+        try:
+            with (
+                patch.object(preflight_checks, 'IfoeL2ConnectivityCheck') as checker_cls,
+                patch.object(preflight_checks, 'preflight_update_test_result'),
+            ):
+                checker_cls.return_value.run.return_value = checker_results
+                preflight_checks.test_ifoe_l2_connectivity(phdl, config, cluster)
+
+            kwargs = checker_cls.call_args.kwargs
+            self.assertEqual(kwargs['ssh_timeout'], 900)
+            self.assertEqual(kwargs['loss_threshold_pct'], 3.0)
+            result = preflight_checks.preflight_results['ifoe_l2_connectivity']
+            self.assertEqual(result['ping_timeout'], 900)
+            self.assertEqual(result['loss_threshold_pct'], 3.0)
+        finally:
+            preflight_checks.preflight_results.clear()
+            preflight_checks.preflight_results.update(previous_results)
+
     def test_disabled_l2ping_skips_without_constructing_checker(self):
         from cvs.tests.preflight import preflight_checks
 

cvs/parsers/schemas.py

@@ -1373,6 +1373,17 @@ class PreflightL2PingConfig(BaseModel):
 
     enabled: bool = Field(default=False, description="Enable the mandatory IFoE L2 connectivity gate")
     pings_per_port: int = Field(default=3, ge=1, description="Ping samples per selected IFoE port pair")
+    loss_threshold_pct: float = Field(
+        default=0.0,
+        ge=0.0,
+        le=100.0,
+        description="Maximum tolerated packet loss percentage per traffic type",
+    )
+    ping_timeout: int = Field(
+        default=600,
+        ge=30,
+        description="PSSH read timeout in seconds for each afmctl ping invocation",
+    )
 
 
 class PreflightTransferBenchConfig(BaseModel):

cvs/tests/preflight/preflight_checks.py

@@ -815,7 +815,11 @@ def _l2ping_config(config_dict):
     config = _ifoe_config(config_dict).get('l2ping', {})
     if not isinstance(config, dict):
         raise ValueError("preflight.connectivity_check.ifoe.l2ping must be an object")
-    unknown = sorted(key for key in set(config) - {'enabled', 'pings_per_port'} if not key.startswith('_'))
+    unknown = sorted(
+        key
+        for key in set(config) - {'enabled', 'pings_per_port', 'loss_threshold_pct', 'ping_timeout'}
+        if not key.startswith('_')
+    )
     if unknown:
         raise ValueError("Unsupported preflight.connectivity_check.ifoe.l2ping option(s): " + ', '.join(unknown))
     return config
@@ -889,9 +893,19 @@ def _run_ifoe_l2_connectivity(phdl, config_dict, cluster_dict):
     if pings_per_port < 1:
         raise ValueError("preflight.connectivity_check.ifoe.l2ping.pings_per_port must be at least 1")
 
+    loss_threshold_pct = float(l2ping_config.get('loss_threshold_pct', 0.0))
+    if loss_threshold_pct < 0.0 or loss_threshold_pct > 100.0:
+        raise ValueError("preflight.connectivity_check.ifoe.l2ping.loss_threshold_pct must be between 0 and 100")
+
+    ping_timeout = int(l2ping_config.get('ping_timeout', 600))
+    if ping_timeout < 30:
+        raise ValueError("preflight.connectivity_check.ifoe.l2ping.ping_timeout must be at least 30 seconds")
+
     log.info(
-        "Running strict IFoE L2 full-mesh connectivity (pings_per_port=%d) on %d host(s)",
+        "Running strict IFoE L2 full-mesh connectivity (pings_per_port=%d, ping_timeout=%ds, loss_threshold_pct=%.2f) on %d host(s)",
         pings_per_port,
+        ping_timeout,
+        loss_threshold_pct,
         len(phdl.reachable_hosts),
     )
 
@@ -906,8 +920,8 @@ def _run_ifoe_l2_connectivity(phdl, config_dict, cluster_dict):
         pings_per_port=pings_per_port,
         per_ping_timeout=None,
         traffic_types=['ifoe_req', 'ifoe_resp', 'non_ifoe'],
-        loss_threshold_pct=0.0,
-        ssh_timeout=180,
+        loss_threshold_pct=loss_threshold_pct,
+        ssh_timeout=ping_timeout,
         use_sudo=True,
         json_args=['--json'],
         allow_text_fallback=False,
@@ -947,7 +961,8 @@ def _run_ifoe_l2_connectivity(phdl, config_dict, cluster_dict):
         'total_invocations': total_invocations,
         'failed_invocations': failed_invocations,
         'pings_per_port': pings_per_port,
-        'loss_threshold_pct': 0.0,
+        'loss_threshold_pct': loss_threshold_pct,
+        'ping_timeout': ping_timeout,
         'traffic_types': ['ifoe_req', 'ifoe_resp', 'non_ifoe'],
         'mesh_mode': 'full_mesh',
         'ports': 'up',