From af4adcf0777e5c5ff73209b38c3b842d467d2114 Mon Sep 17 00:00:00 2001 From: Jake Ororke Date: Wed, 14 Oct 2026 09:00:37 -0700 Subject: [PATCH 1/5] Classify Python tests by the SDK device-requirement marker The SDK now declares each test's required DUT state with a marker base class and is dropping is_commissioning=True from step 1. Report the marker from test_harness_client.py and use it to pick COMMISSIONING or NO_COMMISSIONING, falling back to the step-1 flag for unmarked tests and SDKs without markers. This is a companion change to work with changes made in SDK PR https://github.com/project-chip/connectedhomeip/pull/73565 --- .../models/python_test_parser.py | 52 +++++- .../models/rpc_client/test_harness_client.py | 27 +++- .../tests/python_tests/test_python_parser.py | 38 +++++ .../python_tests_info_markers.json | 153 ++++++++++++++++++ 4 files changed, 262 insertions(+), 8 deletions(-) create mode 100644 test_collections/matter/sdk_tests/support/tests/python_tests/test_python_script/python_tests_info_markers.json diff --git a/test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py b/test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py index 59b9434a..e4fbfca2 100644 --- a/test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py +++ b/test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py @@ -41,6 +41,16 @@ "TC_IDM_12_1", ] +# Whether each SDK device-requirement marker (matter.testing.matter_testing) needs the +# DUT commissioned before the test runs. Tests without a marker, or from SDKs that +# predate the markers, fall back to the first step's is_commissioning flag. +device_requirement_needs_commissioning = { + "MatterTestCommissionedDevice": True, + "MatterTestUncommissionedDevice": False, + "MatterTestCommissioner": False, + "CertificationUnitTestNoDevice": False, +} + def parse_python_script(path: Path) -> list[PythonTest]: """Parse a python file into a list of PythonTest models. @@ -82,14 +92,11 @@ def parse_python_script(path: Path) -> list[PythonTest]: test_class_name = script_info["class_name"] parsed_steps = script_info["steps"] - is_commssioning = False + is_commssioning = __needs_commissioning(script_info, test_name) test_steps: list[MatterTestStep] = [] - for index, step in enumerate(parsed_steps): + for step in parsed_steps: step_description = step["description"] - if index == 0: - is_commssioning = step["is_commissioning"] - test_steps.append( MatterTestStep( label=step_description, @@ -100,8 +107,9 @@ def parse_python_script(path: Path) -> list[PythonTest]: # - PythonTestType.MANDATORY: Mandatory test cases # - PythonTestType.LEGACY: Tests that have only one step and with this # name: "Run entire test" - # - PythonTestType.COMMISSIONING: Test cases flagged as commissioning - # - PythonTestType.NO_COMMISSIONING: Test cases flagged as no commissioning + # - PythonTestType.COMMISSIONING: Test cases whose device-requirement + # marker, or first step's is_commissioning flag, needs a commissioned DUT + # - PythonTestType.NO_COMMISSIONING: All other test cases if test_name in mandatory_python_tcs_public_id: python_test_type = PythonTestType.MANDATORY elif len(test_steps) == 1 and test_steps[0].label == "Run entire test": @@ -192,6 +200,36 @@ def __test_case_name(function_name: str) -> Optional[str]: return None +def __needs_commissioning(script_info: dict[str, Any], test_name: str) -> bool: + """Whether the DUT must be commissioned before the test runs. + + The test class's device-requirement marker decides when present. Otherwise the + first step's is_commissioning flag decides, as for SDKs without markers and for + tests that declare no marker. + + Args: + script_info (dict[str, Any]): Test info entry from the parsed JSON file. + test_name (str): Test case name, used for logging. + + Returns: + bool: True if the DUT must be commissioned before the test runs. + """ + steps = script_info["steps"] + step_flag = bool(steps[0]["is_commissioning"]) if steps else False + + marker = script_info.get("device_requirement") + if marker not in device_requirement_needs_commissioning: + return step_flag + + needs_commissioning = device_requirement_needs_commissioning[marker] + if step_flag and not needs_commissioning: + logger.warning( + f"{test_name}: first step sets is_commissioning but the test class is " + f"{marker}. Using {marker}." + ) + return needs_commissioning + + def __parse_test_case( tc_name: str, methods: list[FunctionDefType], class_name: str, path: Path ) -> PythonTest: diff --git a/test_collections/matter/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py b/test_collections/matter/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py index f8d720f2..b84eba36 100644 --- a/test_collections/matter/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py +++ b/test_collections/matter/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py @@ -33,6 +33,11 @@ run_tests, ) +try: + from matter.testing.matter_testing import device_requirement +except ImportError: # SDKs before project-chip/connectedhomeip#73214 have no markers + device_requirement = None + # TH-specific argument keys TH_ARG_GET_TEST_INFO = "get-test-info" TH_ARG_TH_CLIENT_TEST = "th-client-test" @@ -215,11 +220,31 @@ def main() -> None: ) +def _device_requirement_name(test_class) -> str | None: + """Name of the SDK device-requirement marker the test class inherits, if any. + + None when the SDK predates the markers, the class declares no marker (e.g. + BasicCompositionTests-based tests), or it declares conflicting markers. The + parser then falls back to the first step's is_commissioning flag. + """ + if device_requirement is None: + return None + try: + marker = device_requirement(test_class) + except ValueError: + return None + return marker.__name__ if marker is not None else None + + def get_test_info_support(script_path: str, class_name: str, config: MatterTestConfig): module = importlib.import_module(script_path.replace("/", ".")) TestClassReference = getattr(module, class_name) test_info = get_test_info(TestClassReference, config) - return json.loads(json.dumps(test_info, default=lambda o: o.__dict__)) + info = json.loads(json.dumps(test_info, default=lambda o: o.__dict__)) + requirement = _device_requirement_name(TestClassReference) + for test in info: + test["device_requirement"] = requirement + return info def run_test(script_path: str, class_name: str, config: MatterTestConfig) -> None: diff --git a/test_collections/matter/sdk_tests/support/tests/python_tests/test_python_parser.py b/test_collections/matter/sdk_tests/support/tests/python_tests/test_python_parser.py index 299853c9..937f1125 100644 --- a/test_collections/matter/sdk_tests/support/tests/python_tests/test_python_parser.py +++ b/test_collections/matter/sdk_tests/support/tests/python_tests/test_python_parser.py @@ -16,7 +16,10 @@ # flake8: noqa # Ignore flake8 check for this file from pathlib import Path +from unittest import mock +from ...python_testing.models import python_test_parser +from ...python_testing.models.python_test_models import PythonTestType from ...python_testing.models.python_test_parser import parse_python_script @@ -34,3 +37,38 @@ def test_python_file_parser() -> None: assert len(tests[0].PICS) == 2 count = sum(1 for test in tests if str(test.path) == "sdk/TC_SameClass") assert count == 2 + + +def test_python_parser_device_requirement() -> None: + """The device-requirement marker decides the test type when present, and the + first step's is_commissioning flag decides otherwise.""" + parent_path = Path(__file__).parent + file_path = parent_path.joinpath( + "test_python_script/python_tests_info_markers.json" + ) + + with mock.patch.object(python_test_parser, "logger") as mock_logger: + tests = {test.name: test for test in parse_python_script(file_path)} + + expected_types = { + "TC_Marker_Commissioned": PythonTestType.COMMISSIONING, + "TC_Marker_Commissioner": PythonTestType.NO_COMMISSIONING, + "TC_Marker_Uncommissioned": PythonTestType.NO_COMMISSIONING, + # No marker, or JSON from an SDK without markers: the step flag decides + "TC_No_Marker_Flagged": PythonTestType.COMMISSIONING, + "TC_Old_Json_Flagged": PythonTestType.COMMISSIONING, + "TC_Old_Json_Unflagged": PythonTestType.NO_COMMISSIONING, + # The marker wins over a contradicting step flag + "TC_Contradiction": PythonTestType.NO_COMMISSIONING, + # LEGACY and MANDATORY still take precedence over the marker + "TC_Marker_Legacy": PythonTestType.LEGACY, + "TC_IDM_12_1": PythonTestType.MANDATORY, + } + assert { + name: test.python_test_type for name, test in tests.items() + } == expected_types + + assert all(step.is_commissioning for step in tests["TC_Marker_Commissioned"].steps) + + mock_logger.warning.assert_called_once() + assert "TC_Contradiction" in mock_logger.warning.call_args.args[0] diff --git a/test_collections/matter/sdk_tests/support/tests/python_tests/test_python_script/python_tests_info_markers.json b/test_collections/matter/sdk_tests/support/tests/python_tests/test_python_script/python_tests_info_markers.json new file mode 100644 index 00000000..25e70e65 --- /dev/null +++ b/test_collections/matter/sdk_tests/support/tests/python_tests/test_python_script/python_tests_info_markers.json @@ -0,0 +1,153 @@ +{ + "sdk_sha": "aabbccddeeff11223344556677889900", + "tests": [ + { + "class_name": "TC_Marker_Commissioned", + "desc": "Commissioned marker without a step flag", + "device_requirement": "MatterTestCommissionedDevice", + "function": "test_TC_Marker_Commissioned", + "path": "sdk/TC_Marker_Commissioned", + "pics": [], + "steps": [ + { + "description": "Commissioning, already done", + "expectation": "", + "is_commissioning": false, + "test_plan_number": 1 + }, + { + "description": "Second step", + "expectation": "", + "is_commissioning": false, + "test_plan_number": 2 + } + ] + }, + { + "class_name": "TC_Marker_Commissioner", + "desc": "Commissioner marker", + "device_requirement": "MatterTestCommissioner", + "function": "test_TC_Marker_Commissioner", + "path": "sdk/TC_Marker_Commissioner", + "pics": [], + "steps": [ + { + "description": "TH commissions the DUT", + "expectation": "", + "is_commissioning": false, + "test_plan_number": 1 + } + ] + }, + { + "class_name": "TC_Marker_Uncommissioned", + "desc": "Uncommissioned marker", + "device_requirement": "MatterTestUncommissionedDevice", + "function": "test_TC_Marker_Uncommissioned", + "path": "sdk/TC_Marker_Uncommissioned", + "pics": [], + "steps": [ + { + "description": "TH discovers the commissionable DUT", + "expectation": "", + "is_commissioning": false, + "test_plan_number": 1 + } + ] + }, + { + "class_name": "TC_No_Marker_Flagged", + "desc": "No marker, first step flagged", + "device_requirement": null, + "function": "test_TC_No_Marker_Flagged", + "path": "sdk/TC_No_Marker_Flagged", + "pics": [], + "steps": [ + { + "description": "Commissioning, already done", + "expectation": "", + "is_commissioning": true, + "test_plan_number": 1 + } + ] + }, + { + "class_name": "TC_Old_Json_Flagged", + "desc": "No device_requirement key, first step flagged", + "function": "test_TC_Old_Json_Flagged", + "path": "sdk/TC_Old_Json_Flagged", + "pics": [], + "steps": [ + { + "description": "Commissioning, already done", + "expectation": "", + "is_commissioning": true, + "test_plan_number": 1 + } + ] + }, + { + "class_name": "TC_Old_Json_Unflagged", + "desc": "No device_requirement key, first step not flagged", + "function": "test_TC_Old_Json_Unflagged", + "path": "sdk/TC_Old_Json_Unflagged", + "pics": [], + "steps": [ + { + "description": "First step", + "expectation": "", + "is_commissioning": false, + "test_plan_number": 1 + } + ] + }, + { + "class_name": "TC_Contradiction", + "desc": "Commissioner marker but first step flagged", + "device_requirement": "MatterTestCommissioner", + "function": "test_TC_Contradiction", + "path": "sdk/TC_Contradiction", + "pics": [], + "steps": [ + { + "description": "Commissioning, already done", + "expectation": "", + "is_commissioning": true, + "test_plan_number": 1 + } + ] + }, + { + "class_name": "TC_Marker_Legacy", + "desc": "Commissioned marker without defined steps", + "device_requirement": "MatterTestCommissionedDevice", + "function": "test_TC_Marker_Legacy", + "path": "sdk/TC_Marker_Legacy", + "pics": [], + "steps": [ + { + "description": "Run entire test", + "expectation": "", + "is_commissioning": false, + "test_plan_number": 1 + } + ] + }, + { + "class_name": "TC_IDM_12_1", + "desc": "Mandatory test with a commissioned marker", + "device_requirement": "MatterTestCommissionedDevice", + "function": "test_TC_IDM_12_1", + "path": "sdk/TC_DeviceBasicComposition", + "pics": [], + "steps": [ + { + "description": "Commissioning, already done", + "expectation": "", + "is_commissioning": false, + "test_plan_number": 1 + } + ] + } + ] +} From d04f499510c660653f21eda8eb4e0162af8b32b5 Mon Sep 17 00:00:00 2001 From: Jake Ororke Date: Wed, 14 Oct 2026 13:18:48 -0700 Subject: [PATCH 2/5] Fix is_commissioning typo in parse_python_script --- .../support/python_testing/models/python_test_parser.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py b/test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py index e4fbfca2..2f8bf617 100644 --- a/test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py +++ b/test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py @@ -92,7 +92,7 @@ def parse_python_script(path: Path) -> list[PythonTest]: test_class_name = script_info["class_name"] parsed_steps = script_info["steps"] - is_commssioning = __needs_commissioning(script_info, test_name) + is_commissioning = __needs_commissioning(script_info, test_name) test_steps: list[MatterTestStep] = [] for step in parsed_steps: step_description = step["description"] @@ -100,7 +100,7 @@ def parse_python_script(path: Path) -> list[PythonTest]: test_steps.append( MatterTestStep( label=step_description, - is_commissioning=is_commssioning, + is_commissioning=is_commissioning, ) ) @@ -114,7 +114,7 @@ def parse_python_script(path: Path) -> list[PythonTest]: python_test_type = PythonTestType.MANDATORY elif len(test_steps) == 1 and test_steps[0].label == "Run entire test": python_test_type = PythonTestType.LEGACY - elif is_commssioning: + elif is_commissioning: python_test_type = PythonTestType.COMMISSIONING else: python_test_type = PythonTestType.NO_COMMISSIONING From 263bcc992664d4e8c0ff482d74bf861df9b1b34e Mon Sep 17 00:00:00 2001 From: Jake Ororke Date: Wed, 14 Oct 2026 13:27:36 -0700 Subject: [PATCH 3/5] Warn on unknown device-requirement markers --- .../python_testing/models/python_test_parser.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py b/test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py index 2f8bf617..e6a62385 100644 --- a/test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py +++ b/test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py @@ -42,8 +42,9 @@ ] # Whether each SDK device-requirement marker (matter.testing.matter_testing) needs the -# DUT commissioned before the test runs. Tests without a marker, or from SDKs that -# predate the markers, fall back to the first step's is_commissioning flag. +# DUT commissioned before the test runs. Tests without a marker, from SDKs that +# predate the markers, or with a marker missing here, fall back to the first step's +# is_commissioning flag. device_requirement_needs_commissioning = { "MatterTestCommissionedDevice": True, "MatterTestUncommissionedDevice": False, @@ -205,7 +206,8 @@ def __needs_commissioning(script_info: dict[str, Any], test_name: str) -> bool: The test class's device-requirement marker decides when present. Otherwise the first step's is_commissioning flag decides, as for SDKs without markers and for - tests that declare no marker. + tests that declare no marker. A marker this TH doesn't know also falls back to + the flag, with a warning. Args: script_info (dict[str, Any]): Test info entry from the parsed JSON file. @@ -218,7 +220,13 @@ def __needs_commissioning(script_info: dict[str, Any], test_name: str) -> bool: step_flag = bool(steps[0]["is_commissioning"]) if steps else False marker = script_info.get("device_requirement") + if marker is None: + return step_flag if marker not in device_requirement_needs_commissioning: + logger.warning( + f"{test_name}: unknown device-requirement marker {marker}. Using the " + "first step's is_commissioning flag." + ) return step_flag needs_commissioning = device_requirement_needs_commissioning[marker] From 47470db8a5149997c5d05f9c9f8a955376133b46 Mon Sep 17 00:00:00 2001 From: Jake Ororke Date: Wed, 14 Oct 2026 14:25:26 -0700 Subject: [PATCH 4/5] Treat a missing step is_commissioning as unflagged and test the marker warnings --- .../models/python_test_parser.py | 2 +- .../tests/python_tests/test_python_parser.py | 13 ++++++-- .../python_tests_info_markers.json | 31 +++++++++++++++++++ 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py b/test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py index e6a62385..c652571c 100644 --- a/test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py +++ b/test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py @@ -217,7 +217,7 @@ def __needs_commissioning(script_info: dict[str, Any], test_name: str) -> bool: bool: True if the DUT must be commissioned before the test runs. """ steps = script_info["steps"] - step_flag = bool(steps[0]["is_commissioning"]) if steps else False + step_flag = bool(steps[0].get("is_commissioning", False)) if steps else False marker = script_info.get("device_requirement") if marker is None: diff --git a/test_collections/matter/sdk_tests/support/tests/python_tests/test_python_parser.py b/test_collections/matter/sdk_tests/support/tests/python_tests/test_python_parser.py index 937f1125..e4de88b5 100644 --- a/test_collections/matter/sdk_tests/support/tests/python_tests/test_python_parser.py +++ b/test_collections/matter/sdk_tests/support/tests/python_tests/test_python_parser.py @@ -58,6 +58,10 @@ def test_python_parser_device_requirement() -> None: "TC_No_Marker_Flagged": PythonTestType.COMMISSIONING, "TC_Old_Json_Flagged": PythonTestType.COMMISSIONING, "TC_Old_Json_Unflagged": PythonTestType.NO_COMMISSIONING, + # A step without the is_commissioning key counts as unflagged + "TC_No_Step_Flag_Key": PythonTestType.NO_COMMISSIONING, + # A marker the TH doesn't know: the step flag decides + "TC_Unknown_Marker": PythonTestType.COMMISSIONING, # The marker wins over a contradicting step flag "TC_Contradiction": PythonTestType.NO_COMMISSIONING, # LEGACY and MANDATORY still take precedence over the marker @@ -70,5 +74,10 @@ def test_python_parser_device_requirement() -> None: assert all(step.is_commissioning for step in tests["TC_Marker_Commissioned"].steps) - mock_logger.warning.assert_called_once() - assert "TC_Contradiction" in mock_logger.warning.call_args.args[0] + warnings = [call.args[0] for call in mock_logger.warning.call_args_list] + assert len(warnings) == 2 + assert any("TC_Contradiction" in warning for warning in warnings) + assert any( + "TC_Unknown_Marker" in warning and "MatterTestFutureDevice" in warning + for warning in warnings + ) diff --git a/test_collections/matter/sdk_tests/support/tests/python_tests/test_python_script/python_tests_info_markers.json b/test_collections/matter/sdk_tests/support/tests/python_tests/test_python_script/python_tests_info_markers.json index 25e70e65..363d60d1 100644 --- a/test_collections/matter/sdk_tests/support/tests/python_tests/test_python_script/python_tests_info_markers.json +++ b/test_collections/matter/sdk_tests/support/tests/python_tests/test_python_script/python_tests_info_markers.json @@ -101,6 +101,37 @@ } ] }, + { + "class_name": "TC_No_Step_Flag_Key", + "desc": "No marker, first step without an is_commissioning key", + "device_requirement": null, + "function": "test_TC_No_Step_Flag_Key", + "path": "sdk/TC_No_Step_Flag_Key", + "pics": [], + "steps": [ + { + "description": "First step", + "expectation": "", + "test_plan_number": 1 + } + ] + }, + { + "class_name": "TC_Unknown_Marker", + "desc": "Marker unknown to the TH, first step flagged", + "device_requirement": "MatterTestFutureDevice", + "function": "test_TC_Unknown_Marker", + "path": "sdk/TC_Unknown_Marker", + "pics": [], + "steps": [ + { + "description": "Commissioning, already done", + "expectation": "", + "is_commissioning": true, + "test_plan_number": 1 + } + ] + }, { "class_name": "TC_Contradiction", "desc": "Commissioner marker but first step flagged", From 2905b5aac8db44c446cfe6da42fc4cb1a1cc8944 Mon Sep 17 00:00:00 2001 From: Jake Ororke Date: Thu, 15 Oct 2026 04:55:49 -0700 Subject: [PATCH 5/5] Warn when a test class declares conflicting device-requirement markers --- .../models/python_test_parser.py | 10 +++++++-- .../models/rpc_client/test_harness_client.py | 22 ++++++++++++------- .../tests/python_tests/test_python_parser.py | 9 ++++++-- .../python_tests_info_markers.json | 17 ++++++++++++++ 4 files changed, 46 insertions(+), 12 deletions(-) diff --git a/test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py b/test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py index c652571c..a8cbc225 100644 --- a/test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py +++ b/test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py @@ -206,8 +206,9 @@ def __needs_commissioning(script_info: dict[str, Any], test_name: str) -> bool: The test class's device-requirement marker decides when present. Otherwise the first step's is_commissioning flag decides, as for SDKs without markers and for - tests that declare no marker. A marker this TH doesn't know also falls back to - the flag, with a warning. + tests that declare no marker. A marker this TH doesn't know, or conflicting + markers (reported in device_requirement_error), also fall back to the flag, + with a warning. Args: script_info (dict[str, Any]): Test info entry from the parsed JSON file. @@ -221,6 +222,11 @@ def __needs_commissioning(script_info: dict[str, Any], test_name: str) -> bool: marker = script_info.get("device_requirement") if marker is None: + error = script_info.get("device_requirement_error") + if error: + logger.warning( + f"{test_name}: {error}. Using the first step's is_commissioning flag." + ) return step_flag if marker not in device_requirement_needs_commissioning: logger.warning( diff --git a/test_collections/matter/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py b/test_collections/matter/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py index b84eba36..db6330f0 100644 --- a/test_collections/matter/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py +++ b/test_collections/matter/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py @@ -223,16 +223,14 @@ def main() -> None: def _device_requirement_name(test_class) -> str | None: """Name of the SDK device-requirement marker the test class inherits, if any. - None when the SDK predates the markers, the class declares no marker (e.g. - BasicCompositionTests-based tests), or it declares conflicting markers. The - parser then falls back to the first step's is_commissioning flag. + None when the SDK predates the markers or the class declares no marker (e.g. + BasicCompositionTests-based tests). The parser then falls back to the first + step's is_commissioning flag. Raises ValueError, from the SDK's + device_requirement(), when the class declares conflicting markers. """ if device_requirement is None: return None - try: - marker = device_requirement(test_class) - except ValueError: - return None + marker = device_requirement(test_class) return marker.__name__ if marker is not None else None @@ -241,9 +239,17 @@ def get_test_info_support(script_path: str, class_name: str, config: MatterTestC TestClassReference = getattr(module, class_name) test_info = get_test_info(TestClassReference, config) info = json.loads(json.dumps(test_info, default=lambda o: o.__dict__)) - requirement = _device_requirement_name(TestClassReference) + try: + requirement = _device_requirement_name(TestClassReference) + requirement_error = None + except ValueError as e: + # The TH discards this client's output, so the parser logs the error + requirement = None + requirement_error = str(e) for test in info: test["device_requirement"] = requirement + if requirement_error is not None: + test["device_requirement_error"] = requirement_error return info diff --git a/test_collections/matter/sdk_tests/support/tests/python_tests/test_python_parser.py b/test_collections/matter/sdk_tests/support/tests/python_tests/test_python_parser.py index e4de88b5..ce3503f9 100644 --- a/test_collections/matter/sdk_tests/support/tests/python_tests/test_python_parser.py +++ b/test_collections/matter/sdk_tests/support/tests/python_tests/test_python_parser.py @@ -60,8 +60,9 @@ def test_python_parser_device_requirement() -> None: "TC_Old_Json_Unflagged": PythonTestType.NO_COMMISSIONING, # A step without the is_commissioning key counts as unflagged "TC_No_Step_Flag_Key": PythonTestType.NO_COMMISSIONING, - # A marker the TH doesn't know: the step flag decides + # A marker the TH doesn't know, or conflicting markers: the step flag decides "TC_Unknown_Marker": PythonTestType.COMMISSIONING, + "TC_Conflicting_Markers": PythonTestType.COMMISSIONING, # The marker wins over a contradicting step flag "TC_Contradiction": PythonTestType.NO_COMMISSIONING, # LEGACY and MANDATORY still take precedence over the marker @@ -75,9 +76,13 @@ def test_python_parser_device_requirement() -> None: assert all(step.is_commissioning for step in tests["TC_Marker_Commissioned"].steps) warnings = [call.args[0] for call in mock_logger.warning.call_args_list] - assert len(warnings) == 2 + assert len(warnings) == 3 assert any("TC_Contradiction" in warning for warning in warnings) assert any( "TC_Unknown_Marker" in warning and "MatterTestFutureDevice" in warning for warning in warnings ) + assert any( + "TC_Conflicting_Markers" in warning and "conflicting" in warning + for warning in warnings + ) diff --git a/test_collections/matter/sdk_tests/support/tests/python_tests/test_python_script/python_tests_info_markers.json b/test_collections/matter/sdk_tests/support/tests/python_tests/test_python_script/python_tests_info_markers.json index 363d60d1..0e2b152b 100644 --- a/test_collections/matter/sdk_tests/support/tests/python_tests/test_python_script/python_tests_info_markers.json +++ b/test_collections/matter/sdk_tests/support/tests/python_tests/test_python_script/python_tests_info_markers.json @@ -116,6 +116,23 @@ } ] }, + { + "class_name": "TC_Conflicting_Markers", + "desc": "Conflicting markers reported by the client, first step flagged", + "device_requirement": null, + "device_requirement_error": "TC_Conflicting_Markers declares conflicting device-requirement markers: MatterTestCommissionedDevice, MatterTestCommissioner", + "function": "test_TC_Conflicting_Markers", + "path": "sdk/TC_Conflicting_Markers", + "pics": [], + "steps": [ + { + "description": "Commissioning, already done", + "expectation": "", + "is_commissioning": true, + "test_plan_number": 1 + } + ] + }, { "class_name": "TC_Unknown_Marker", "desc": "Marker unknown to the TH, first step flagged",