Classify Python tests by the SDK device-requirement marker

test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py

@@ -41,6 +41,17 @@
     "TC_IDM_12_1",
 ]
 
+# Whether each SDK device-requirement marker (matter.testing.matter_testing) needs the
+# DUT commissioned before the test runs. Tests without a marker, from SDKs that
+# predate the markers, or with a marker missing here, fall back to the first step's
+# is_commissioning flag.
+device_requirement_needs_commissioning = {
+    "MatterTestCommissionedDevice": True,
+    "MatterTestUncommissionedDevice": False,
+    "MatterTestCommissioner": False,
+    "CertificationUnitTestNoDevice": False,
+}
+
 
 def parse_python_script(path: Path) -> list[PythonTest]:
     """Parse a python file into a list of PythonTest models.
@@ -82,31 +93,29 @@ def parse_python_script(path: Path) -> list[PythonTest]:
             test_class_name = script_info["class_name"]
             parsed_steps = script_info["steps"]
 
-            is_commssioning = False
+            is_commissioning = __needs_commissioning(script_info, test_name)
             test_steps: list[MatterTestStep] = []
-            for index, step in enumerate(parsed_steps):
+            for step in parsed_steps:
                 step_description = step["description"]
 
-                if index == 0:
-                    is_commssioning = step["is_commissioning"]
-
                 test_steps.append(
                     MatterTestStep(
                         label=step_description,
-                        is_commissioning=is_commssioning,
+                        is_commissioning=is_commissioning,
                     )
                 )
 
             # - PythonTestType.MANDATORY: Mandatory test cases
             # - PythonTestType.LEGACY: Tests that have only one step and with this
             #   name: "Run entire test"
-            # - PythonTestType.COMMISSIONING: Test cases flagged as commissioning
-            # - PythonTestType.NO_COMMISSIONING: Test cases flagged as no commissioning
+            # - PythonTestType.COMMISSIONING: Test cases whose device-requirement
+            #   marker, or first step's is_commissioning flag, needs a commissioned DUT
+            # - PythonTestType.NO_COMMISSIONING: All other test cases
             if test_name in mandatory_python_tcs_public_id:
                 python_test_type = PythonTestType.MANDATORY
             elif len(test_steps) == 1 and test_steps[0].label == "Run entire test":
                 python_test_type = PythonTestType.LEGACY
-            elif is_commssioning:
+            elif is_commissioning:
                 python_test_type = PythonTestType.COMMISSIONING
             else:
                 python_test_type = PythonTestType.NO_COMMISSIONING
@@ -192,6 +201,49 @@ def __test_case_name(function_name: str) -> Optional[str]:
     return None
 
 
+def __needs_commissioning(script_info: dict[str, Any], test_name: str) -> bool:
+    """Whether the DUT must be commissioned before the test runs.
+
+    The test class's device-requirement marker decides when present. Otherwise the
+    first step's is_commissioning flag decides, as for SDKs without markers and for
+    tests that declare no marker. A marker this TH doesn't know, or conflicting
+    markers (reported in device_requirement_error), also fall back to the flag,
+    with a warning.
+
+    Args:
+        script_info (dict[str, Any]): Test info entry from the parsed JSON file.
+        test_name (str): Test case name, used for logging.
+
+    Returns:
+        bool: True if the DUT must be commissioned before the test runs.
+    """
+    steps = script_info["steps"]
+    step_flag = bool(steps[0].get("is_commissioning", False)) if steps else False
+
+    marker = script_info.get("device_requirement")
+    if marker is None:
+        error = script_info.get("device_requirement_error")
+        if error:
+            logger.warning(
+                f"{test_name}: {error}. Using the first step's is_commissioning flag."
+            )
+        return step_flag
+    if marker not in device_requirement_needs_commissioning:
+        logger.warning(
+            f"{test_name}: unknown device-requirement marker {marker}. Using the "
+            "first step's is_commissioning flag."
+        )
+        return step_flag
+
+    needs_commissioning = device_requirement_needs_commissioning[marker]
+    if step_flag and not needs_commissioning:
+        logger.warning(
+            f"{test_name}: first step sets is_commissioning but the test class is "
+            f"{marker}. Using {marker}."
+        )
+    return needs_commissioning
+
+
 def __parse_test_case(
     tc_name: str, methods: list[FunctionDefType], class_name: str, path: Path
 ) -> PythonTest:

test_collections/matter/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py

@@ -33,6 +33,11 @@
     run_tests,
 )
 
+try:
+    from matter.testing.matter_testing import device_requirement
+except ImportError:  # SDKs before project-chip/connectedhomeip#73214 have no markers
+    device_requirement = None
+
 # TH-specific argument keys
 TH_ARG_GET_TEST_INFO = "get-test-info"
 TH_ARG_TH_CLIENT_TEST = "th-client-test"
@@ -215,11 +220,37 @@ def main() -> None:
                     )
 
 
+def _device_requirement_name(test_class) -> str | None:
+    """Name of the SDK device-requirement marker the test class inherits, if any.
+
+    None when the SDK predates the markers or the class declares no marker (e.g.
+    BasicCompositionTests-based tests). The parser then falls back to the first
+    step's is_commissioning flag. Raises ValueError, from the SDK's
+    device_requirement(), when the class declares conflicting markers.
+    """
+    if device_requirement is None:
+        return None
+    marker = device_requirement(test_class)
+    return marker.__name__ if marker is not None else None
+
+
 def get_test_info_support(script_path: str, class_name: str, config: MatterTestConfig):
     module = importlib.import_module(script_path.replace("/", "."))
     TestClassReference = getattr(module, class_name)
     test_info = get_test_info(TestClassReference, config)
-    return json.loads(json.dumps(test_info, default=lambda o: o.__dict__))
+    info = json.loads(json.dumps(test_info, default=lambda o: o.__dict__))
+    try:
+        requirement = _device_requirement_name(TestClassReference)
+        requirement_error = None
+    except ValueError as e:
+        # The TH discards this client's output, so the parser logs the error
+        requirement = None
+        requirement_error = str(e)
+    for test in info:
+        test["device_requirement"] = requirement
+        if requirement_error is not None:
+            test["device_requirement_error"] = requirement_error
+    return info
 
 
 def run_test(script_path: str, class_name: str, config: MatterTestConfig) -> None:

test_collections/matter/sdk_tests/support/tests/python_tests/test_python_parser.py

@@ -16,7 +16,10 @@
 # flake8: noqa
 # Ignore flake8 check for this file
 from pathlib import Path
+from unittest import mock
 
+from ...python_testing.models import python_test_parser
+from ...python_testing.models.python_test_models import PythonTestType
 from ...python_testing.models.python_test_parser import parse_python_script
 
 
@@ -34,3 +37,52 @@ def test_python_file_parser() -> None:
     assert len(tests[0].PICS) == 2
     count = sum(1 for test in tests if str(test.path) == "sdk/TC_SameClass")
     assert count == 2
+
+
+def test_python_parser_device_requirement() -> None:
+    """The device-requirement marker decides the test type when present, and the
+    first step's is_commissioning flag decides otherwise."""
+    parent_path = Path(__file__).parent
+    file_path = parent_path.joinpath(
+        "test_python_script/python_tests_info_markers.json"
+    )
+
+    with mock.patch.object(python_test_parser, "logger") as mock_logger:
+        tests = {test.name: test for test in parse_python_script(file_path)}
+
+    expected_types = {
+        "TC_Marker_Commissioned": PythonTestType.COMMISSIONING,
+        "TC_Marker_Commissioner": PythonTestType.NO_COMMISSIONING,
+        "TC_Marker_Uncommissioned": PythonTestType.NO_COMMISSIONING,
+        # No marker, or JSON from an SDK without markers: the step flag decides
+        "TC_No_Marker_Flagged": PythonTestType.COMMISSIONING,
+        "TC_Old_Json_Flagged": PythonTestType.COMMISSIONING,
+        "TC_Old_Json_Unflagged": PythonTestType.NO_COMMISSIONING,
+        # A step without the is_commissioning key counts as unflagged
+        "TC_No_Step_Flag_Key": PythonTestType.NO_COMMISSIONING,
+        # A marker the TH doesn't know, or conflicting markers: the step flag decides
+        "TC_Unknown_Marker": PythonTestType.COMMISSIONING,
+        "TC_Conflicting_Markers": PythonTestType.COMMISSIONING,
+        # The marker wins over a contradicting step flag
+        "TC_Contradiction": PythonTestType.NO_COMMISSIONING,
+        # LEGACY and MANDATORY still take precedence over the marker
+        "TC_Marker_Legacy": PythonTestType.LEGACY,
+        "TC_IDM_12_1": PythonTestType.MANDATORY,
+    }
+    assert {
+        name: test.python_test_type for name, test in tests.items()
+    } == expected_types
+
+    assert all(step.is_commissioning for step in tests["TC_Marker_Commissioned"].steps)
+
+    warnings = [call.args[0] for call in mock_logger.warning.call_args_list]
+    assert len(warnings) == 3
+    assert any("TC_Contradiction" in warning for warning in warnings)
+    assert any(
+        "TC_Unknown_Marker" in warning and "MatterTestFutureDevice" in warning
+        for warning in warnings
+    )
+    assert any(
+        "TC_Conflicting_Markers" in warning and "conflicting" in warning
+        for warning in warnings
+    )