From 5d44087e73647eb44b01d0cd1c3697457913d37b Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Thu, 11 Jun 2026 13:35:28 -0700 Subject: [PATCH 01/78] Add simple copy method to allow simulating removing placeholders. --- tdom/placeholders.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tdom/placeholders.py b/tdom/placeholders.py index 1cf47128..eabe4583 100644 --- a/tdom/placeholders.py +++ b/tdom/placeholders.py @@ -61,6 +61,9 @@ class PlaceholderState: config: PlaceholderConfig = field(default_factory=make_placeholder_config) """Collection of currently 'known and active' placeholder indexes.""" + def copy(self): + return PlaceholderState(known=self.known.copy(), config=self.config) + @property def is_empty(self) -> bool: return len(self.known) == 0 From 4cb21117b49649d1e0e894dd38a490539539ab4f Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Thu, 11 Jun 2026 13:41:36 -0700 Subject: [PATCH 02/78] Typeguard against bogus empty starttag_text. --- tdom/parser.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/tdom/parser.py b/tdom/parser.py index 2eec36d8..63207022 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -189,11 +189,9 @@ def make_open_tag(self, tag: str, attrs: Sequence[HTMLAttribute]) -> OpenTag: # @NOTE: This must be called when the tag is handled since it is # populated based on the most recently finished start tag. Otherwise # the value will be out of sync. - starttag_text = self.get_starttag_text() - if starttag_text is None: - raise AssertionError( - f"Expected startag_text to be set when parsing component at {i_index}." - ) + starttag_text = self.always_get_starttag_text( + f"Expected startag_text to be set when parsing component at {i_index}." + ) tattrs = self.make_tattrs(attrs) @@ -371,6 +369,19 @@ def validate_end_tag(self, tag: str, open_tag: OpenTag) -> int | None: # any of this in the parser, instead relying on higher layers. return tag_ref.i_indexes[0] + def always_get_starttag_text( + self, msg: str = "Expecting starttag text to be set." + ) -> str: + """ + Wrap get_starttag_text and just raise if None is returned. + + Do this so we don't guard for `None` everywhere. + """ + starttag_text = self.get_starttag_text() + if starttag_text is None: + raise AssertionError(msg) + return starttag_text + # ------------------------------------------ # HTMLParser tag callbacks # ------------------------------------------ From 28c6198f0c97bc95274d996653733d760732b256 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Thu, 11 Jun 2026 13:45:25 -0700 Subject: [PATCH 03/78] Add debugging/introspection info to open tags. --- tdom/parser.py | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/tdom/parser.py b/tdom/parser.py index 63207022..3cafc45e 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -27,6 +27,12 @@ @dataclass class OpenTElement: + starttag_text: str + " Entire starttag as parsed, includes placeholders, used for debugging. " + raw_attrs: Sequence[HTMLAttribute] + " Attrs as parsed, includes placeholders, used for debugging. " + startend: bool + " Was parsed as startend tag, ie. , used for debugging. " tag: str attrs: tuple[TAttribute, ...] children: list[TNode] = field(default_factory=list) @@ -39,6 +45,12 @@ class OpenTFragment: @dataclass class OpenTComponent: + starttag_text: str + " Entire starttag as parsed, includes placeholders, used for debugging. " + raw_attrs: Sequence[HTMLAttribute] + " Attrs as parsed, includes placeholders, used for debugging. " + startend: bool + " Was parsed as startend tag, ie. , used for debugging. " start_i_index: int children_start_s_index: int """The strings index where the component's children template starts.""" @@ -159,12 +171,20 @@ def make_tattrs(self, attrs: Sequence[HTMLAttribute]) -> tuple[TAttribute, ...]: # Tag Helpers # ------------------------------------------ - def make_open_tag(self, tag: str, attrs: Sequence[HTMLAttribute]) -> OpenTag: + def make_open_tag( + self, tag: str, attrs: Sequence[HTMLAttribute], startend: bool = False + ) -> OpenTag: """Build an OpenTag from a raw tag and attribute tuples.""" tag_ref = self.placeholders.remove_placeholders(tag) if tag_ref.is_literal: - return OpenTElement(tag=tag, attrs=self.make_tattrs(attrs)) + return OpenTElement( + starttag_text=self.always_get_starttag_text(), + raw_attrs=attrs, + startend=startend, + tag=tag, + attrs=self.make_tattrs(attrs), + ) if not tag_ref.is_singleton: raise ValueError( @@ -203,6 +223,9 @@ def make_open_tag(self, tag: str, attrs: Sequence[HTMLAttribute]) -> OpenTag: ) return OpenTComponent( + starttag_text=starttag_text, + raw_attrs=attrs, + startend=startend, start_i_index=i_index, children_start_s_index=children_start_s_index, offset_into_children_start_s=offset_into_children_start_s, @@ -396,7 +419,7 @@ def handle_starttag(self, tag: str, attrs: Sequence[HTMLAttribute]) -> None: def handle_startendtag(self, tag: str, attrs: Sequence[HTMLAttribute]) -> None: """Dispatch a self-closing tag, `` to specialized handlers.""" - open_tag = self.make_open_tag(tag, attrs) + open_tag = self.make_open_tag(tag, attrs, startend=True) final_tag = self.finalize_tag(open_tag) self.append_child(final_tag) From 45fbef7b17e513a7e0ee9b36396bedd6f4aca812 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Thu, 11 Jun 2026 14:22:13 -0700 Subject: [PATCH 04/78] Improve unclosed tags message for ambiguous slash case. --- tdom/parser.py | 46 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/tdom/parser.py b/tdom/parser.py index 3cafc45e..ce0f40ab 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -471,6 +471,33 @@ def reset(self): self.placeholders = PlaceholderState() self.source = None + def has_ambiguous_forward_slash(self, open_tag: OpenTag) -> bool: + """ + Detect when an unquoted attribute value consumes a trailing "/" that + *might* have been meant to attempt to self-close a tag, ie. "/>". + + This can come up with literal values or values with interpolations. + + Such as "
" or "<{Component} title=test/>". + + Or more often "<{Component} title={title}/>" which should be corrected + with "<{Component} title={title} />". + """ + if isinstance(open_tag, (OpenTElement, OpenTComponent)): + return ( + # has attributes + len(open_tag.raw_attrs) > 0 + # last attr not bare attribute + and open_tag.raw_attrs[-1][1] is not None + # last char of last attr is "/" + and open_tag.raw_attrs[-1][1][-1] == "/" + # parsed starttag ends with "/>" + and open_tag.starttag_text.endswith("/>") + # if parsed as startend then its not ambiguous + and not open_tag.startend + ) + return False + def close(self) -> None: if self.waiting_for_data(): # We apply heuristics here to try to guess why the parser didn't finish. @@ -483,7 +510,24 @@ def close(self) -> None: "Parser expects more data, is the template valid html?" ) if self.stack: - raise ValueError("Invalid HTML structure: unclosed tags remain.") + e = ValueError("Invalid HTML structure: unclosed tags remain.") + # Check for tags that might have meant to self-close but whose + # unquoted last attribute value consumed a "/", ie.
. + parent = self.stack[-1] + if isinstance(parent, (OpenTElement, OpenTComponent)): + if isinstance(parent, OpenTElement): + starttag = parent.tag + elif isinstance(parent, OpenTComponent): + starttag = ( + f"{{{self.get_source().format_starttag(parent.start_i_index)}}}" + ) + if self.has_ambiguous_forward_slash(parent): + e.add_note( + f'Did you mean to quote the last attribute or put a space before "/>" for "<{starttag} .../>"?' + ) + else: + e.add_note(f"Most recently unclosed tag is <{starttag} ...>") + raise e if not self.placeholders.is_empty: raise ValueError("Some placeholders were never resolved.") super().close() From 181ee6b434ba04ace96f0e568489f053d4480dc0 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Fri, 12 Jun 2026 13:26:54 -0700 Subject: [PATCH 05/78] Fix method defn order. --- tdom/parser.py | 54 +++++++++++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/tdom/parser.py b/tdom/parser.py index ce0f40ab..148fa34e 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -405,6 +405,33 @@ def always_get_starttag_text( raise AssertionError(msg) return starttag_text + def has_ambiguous_forward_slash(self, open_tag: OpenTag) -> bool: + """ + Detect when an unquoted attribute value consumes a trailing "/" that + *might* have been meant to attempt to self-close a tag, ie. "/>". + + This can come up with literal values or values with interpolations. + + Such as "
" or "<{Component} title=test/>". + + Or more often "<{Component} title={title}/>" which should be corrected + with "<{Component} title={title} />". + """ + if isinstance(open_tag, (OpenTElement, OpenTComponent)): + return ( + # has attributes + len(open_tag.raw_attrs) > 0 + # last attr not bare attribute + and open_tag.raw_attrs[-1][1] is not None + # last char of last attr is "/" + and open_tag.raw_attrs[-1][1][-1] == "/" + # parsed starttag ends with "/>" + and open_tag.starttag_text.endswith("/>") + # if parsed as startend then its not ambiguous + and not open_tag.startend + ) + return False + # ------------------------------------------ # HTMLParser tag callbacks # ------------------------------------------ @@ -471,33 +498,6 @@ def reset(self): self.placeholders = PlaceholderState() self.source = None - def has_ambiguous_forward_slash(self, open_tag: OpenTag) -> bool: - """ - Detect when an unquoted attribute value consumes a trailing "/" that - *might* have been meant to attempt to self-close a tag, ie. "/>". - - This can come up with literal values or values with interpolations. - - Such as "
" or "<{Component} title=test/>". - - Or more often "<{Component} title={title}/>" which should be corrected - with "<{Component} title={title} />". - """ - if isinstance(open_tag, (OpenTElement, OpenTComponent)): - return ( - # has attributes - len(open_tag.raw_attrs) > 0 - # last attr not bare attribute - and open_tag.raw_attrs[-1][1] is not None - # last char of last attr is "/" - and open_tag.raw_attrs[-1][1][-1] == "/" - # parsed starttag ends with "/>" - and open_tag.starttag_text.endswith("/>") - # if parsed as startend then its not ambiguous - and not open_tag.startend - ) - return False - def close(self) -> None: if self.waiting_for_data(): # We apply heuristics here to try to guess why the parser didn't finish. From fcb9f4cfbb3edc16d8ee44f84b4bf73600f05f23 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Fri, 12 Jun 2026 13:27:56 -0700 Subject: [PATCH 06/78] Always fallback to tag str for error, fixes typecheck. --- tdom/parser.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tdom/parser.py b/tdom/parser.py index 148fa34e..f95bc821 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -515,12 +515,12 @@ def close(self) -> None: # unquoted last attribute value consumed a "/", ie.
. parent = self.stack[-1] if isinstance(parent, (OpenTElement, OpenTComponent)): - if isinstance(parent, OpenTElement): - starttag = parent.tag - elif isinstance(parent, OpenTComponent): + if isinstance(parent, OpenTComponent): starttag = ( f"{{{self.get_source().format_starttag(parent.start_i_index)}}}" ) + else: + starttag = parent.tag if self.has_ambiguous_forward_slash(parent): e.add_note( f'Did you mean to quote the last attribute or put a space before "/>" for "<{starttag} .../>"?' From 8b0638e045a90bc7e1b1e79ff1144296b96dd70a Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Fri, 12 Jun 2026 14:38:36 -0700 Subject: [PATCH 07/78] Refine error messages for other cases with trailing slash is consumed instead of self-closing. --- tdom/parser.py | 49 +++++++++++++++++++++++++++++++++++++++------ tdom/parser_test.py | 40 +++++++++++++++++++++++++++++++++--- 2 files changed, 80 insertions(+), 9 deletions(-) diff --git a/tdom/parser.py b/tdom/parser.py index f95bc821..35d5e43c 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -84,6 +84,26 @@ class SourceTracker: def interpolations(self) -> tuple[Interpolation, ...]: return self.template.interpolations + def _check_indices(self, index1: int, index2: int): + last_index = len(self.interpolations) - 1 + if max(index1, index2) > last_index or min(index1, index2) < 0: + raise ValueError( + f"Interpolation indices exceed bounds: {index1} {index2}: [0...{last_index}]" + ) + + def expressions_match(self, i_index1: int, i_index2: int) -> bool: + self._check_indices(i_index1, i_index2) + return ( + self.interpolations[i_index1].expression + == self.interpolations[i_index2].expression + ) + + def values_match(self, i_index1: int, i_index2: int) -> bool: + self._check_indices(i_index1, i_index2) + return ( + self.interpolations[i_index1].value == self.interpolations[i_index2].value + ) + def advance_interpolation(self) -> int: """Call before processing an interpolation to move to the next one.""" self.i_index += 1 @@ -360,7 +380,7 @@ def extract_component_children_ref( def validate_end_tag(self, tag: str, open_tag: OpenTag) -> int | None: """Validate that closing tag matches open tag. Return component end index if applicable.""" - assert self.source, "Parser source tracker not initialized." + source = self.get_source() tag_ref = self.placeholders.remove_placeholders(tag) match open_tag: @@ -380,16 +400,33 @@ def validate_end_tag(self, tag: str, open_tag: OpenTag) -> int | None: case OpenTComponent(start_i_index=start_i_index): if tag_ref.is_literal: - raise ValueError( - f"Mismatched closing tag for component starting at {self.source.format_starttag(start_i_index)}." + starttag = source.format_starttag(start_i_index) + e = ValueError( + f"Mismatched closing tag for component with tag {{{starttag}}}." ) + if self.has_ambiguous_forward_slash(open_tag): + e.add_note( + f'Did you mean to quote the last attribute or put a space before "/>" for "<{{{starttag}}} .../>"?' + ) + raise e if not tag_ref.is_singleton: raise ValueError( "Component end tags must have exactly one interpolation." ) - # HERE BE DRAGONS: the interpolation at end_i_index shuld be a - # component callable that matches the start tag. We do not check - # any of this in the parser, instead relying on higher layers. + if not source.expressions_match( + open_tag.start_i_index, tag_ref.i_indexes[0] + ) and not source.values_match( + open_tag.start_i_index, tag_ref.i_indexes[0] + ): + e = TypeError( + "Component start and end tags must contain the same callable." + ) + if self.has_ambiguous_forward_slash(open_tag): + starttag = source.format_starttag(start_i_index) + e.add_note( + f'Did you mean to quote the last attribute or put a space before "/>" for "<{{{starttag}}} .../>"?' + ) + raise e return tag_ref.i_indexes[0] def always_get_starttag_text( diff --git a/tdom/parser_test.py b/tdom/parser_test.py index d1650ae0..e457a3a2 100644 --- a/tdom/parser_test.py +++ b/tdom/parser_test.py @@ -425,15 +425,17 @@ def Component(): assert node == TComponent(start_i_index=0, end_i_index=1) -def test_component_element_special_case_mismatched_closing_tag_still_parses(): +def test_component_element_special_case_mismatched_closing_tag_error(): def Component1(): pass def Component2(): pass - node = TemplateParser.parse(t"<{Component1}>") - assert node == TComponent(start_i_index=0, end_i_index=1) + with pytest.raises( + TypeError, match="Component start and end tags must contain the same callable." + ): + _ = TemplateParser.parse(t"<{Component1}>") def test_component_element_invalid_closing_tag(): @@ -602,3 +604,35 @@ def test_extract_with_templated_attr_gt_char(self, Component): strings=("
Hello, World!
",), i_indexes=() ), ) + + +class TestComponentUnquotedAttrValue: + @pytest.fixture + def Comp(self): + def _Comp(children: Template, title: str) -> Template: + return children + + return _Comp + + @pytest.fixture + def Comp2(self): + def _Comp2(children: Template, title: str) -> Template: + return children + + return _Comp2 + + def test_comp_unquoted_attr_value_error_root(self, Comp): + with pytest.raises( + ValueError, match="Did you mean to quote the last attribute" + ): + _ = TemplateParser.parse(t"<{Comp} title=today/>") + + def test_comp_unquoted_attr_value_error_nested_in_el(self, Comp): + with pytest.raises( + ValueError, match="Did you mean to quote the last attribute" + ): + _ = TemplateParser.parse(t"
<{Comp} title=today/>
") + + def test_comp_unquoted_attr_value_error_nested_in_comp(self, Comp, Comp2): + with pytest.raises(TypeError, match="Did you mean to quote the last attribute"): + _ = TemplateParser.parse(t"<{Comp2}><{Comp} title=today/>") From 2a1f10a9c839ef010b94a9c522b1bdd6a32abad6 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Fri, 12 Jun 2026 14:52:40 -0700 Subject: [PATCH 08/78] Use getter directly but still guard against None. --- tdom/parser.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tdom/parser.py b/tdom/parser.py index 35d5e43c..16b9719f 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -199,7 +199,7 @@ def make_open_tag( if tag_ref.is_literal: return OpenTElement( - starttag_text=self.always_get_starttag_text(), + starttag_text=self.get_starttag_text(), raw_attrs=attrs, startend=startend, tag=tag, @@ -229,7 +229,7 @@ def make_open_tag( # @NOTE: This must be called when the tag is handled since it is # populated based on the most recently finished start tag. Otherwise # the value will be out of sync. - starttag_text = self.always_get_starttag_text( + starttag_text = self.get_starttag_text( f"Expected startag_text to be set when parsing component at {i_index}." ) @@ -429,15 +429,13 @@ def validate_end_tag(self, tag: str, open_tag: OpenTag) -> int | None: raise e return tag_ref.i_indexes[0] - def always_get_starttag_text( - self, msg: str = "Expecting starttag text to be set." - ) -> str: + def get_starttag_text(self, msg: str = "Expecting starttag text to be set.") -> str: """ Wrap get_starttag_text and just raise if None is returned. Do this so we don't guard for `None` everywhere. """ - starttag_text = self.get_starttag_text() + starttag_text = super().get_starttag_text() if starttag_text is None: raise AssertionError(msg) return starttag_text From 8c617dc673def8c58898a479c3f6930c9504c865 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Fri, 12 Jun 2026 15:07:54 -0700 Subject: [PATCH 09/78] Restrict self-close suggestion to components. --- tdom/parser.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/tdom/parser.py b/tdom/parser.py index 16b9719f..0960ad38 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -549,19 +549,16 @@ def close(self) -> None: # Check for tags that might have meant to self-close but whose # unquoted last attribute value consumed a "/", ie.
. parent = self.stack[-1] - if isinstance(parent, (OpenTElement, OpenTComponent)): - if isinstance(parent, OpenTComponent): - starttag = ( - f"{{{self.get_source().format_starttag(parent.start_i_index)}}}" - ) - else: - starttag = parent.tag - if self.has_ambiguous_forward_slash(parent): - e.add_note( - f'Did you mean to quote the last attribute or put a space before "/>" for "<{starttag} .../>"?' - ) - else: - e.add_note(f"Most recently unclosed tag is <{starttag} ...>") + # @TODO: We need to determine which tags this might apply to, this only applies to components. + if isinstance(parent, OpenTComponent) and self.has_ambiguous_forward_slash( + parent + ): + starttag = ( + f"{{{self.get_source().format_starttag(parent.start_i_index)}}}" + ) + e.add_note( + f'Did you mean to quote the last attribute or put a space before "/>" for "<{starttag} .../>"?' + ) raise e if not self.placeholders.is_empty: raise ValueError("Some placeholders were never resolved.") From c1b0132e0151b2b58183284b1bf5379d4bb6ebca Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Mon, 15 Jun 2026 10:00:21 -0700 Subject: [PATCH 10/78] Restore component start/tag mismatch test. --- tdom/parser_test.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tdom/parser_test.py b/tdom/parser_test.py index e457a3a2..ad923223 100644 --- a/tdom/parser_test.py +++ b/tdom/parser_test.py @@ -425,17 +425,15 @@ def Component(): assert node == TComponent(start_i_index=0, end_i_index=1) -def test_component_element_special_case_mismatched_closing_tag_error(): +def test_component_element_special_case_mismatched_closing_tag_still_parses(): def Component1(): pass def Component2(): pass - with pytest.raises( - TypeError, match="Component start and end tags must contain the same callable." - ): - _ = TemplateParser.parse(t"<{Component1}>") + node = TemplateParser.parse(t"<{Component1}>") + assert node == TComponent(start_i_index=0, end_i_index=1) def test_component_element_invalid_closing_tag(): From 42dcdb2db45f504bb2150156a85c47d9fee03cf4 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Mon, 15 Jun 2026 10:01:53 -0700 Subject: [PATCH 11/78] Add tmap and format_endtag for error reporting. --- tdom/parser.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tdom/parser.py b/tdom/parser.py index 0960ad38..9e36750a 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -25,7 +25,7 @@ type HTMLAttributesDict = dict[str, str | None] -@dataclass +@dataclass(frozen=True) class OpenTElement: starttag_text: str " Entire starttag as parsed, includes placeholders, used for debugging. " @@ -38,12 +38,12 @@ class OpenTElement: children: list[TNode] = field(default_factory=list) -@dataclass +@dataclass(frozen=True) class OpenTFragment: children: list[TNode] = field(default_factory=list) -@dataclass +@dataclass(frozen=True) class OpenTComponent: starttag_text: str " Entire starttag as parsed, includes placeholders, used for debugging. " @@ -127,12 +127,17 @@ def format_starttag(self, i_index: int) -> str: """Format a component start tag for error messages.""" return self.get_expression(i_index, fallback_prefix="component-starttag") + def format_endtag(self, i_index: int) -> str: + return self.get_expression(i_index, fallback_prefix="component-endtag") + class TemplateParser(HTMLParser): root: OpenTFragment stack: list[OpenTag] placeholders: PlaceholderState source: SourceTracker | None + tmap: dict[TComponent | TElement | TFragment, OpenTag] + " Map from completed tnodes back to their opentag for error reporting. " def __init__(self, *, convert_charrefs: bool = True): # This calls HTMLParser.reset() which we override to set up our state. @@ -304,9 +309,13 @@ def finalize_tag( """Finalize an OpenTag into a TNode.""" match open_tag: case OpenTElement(tag=tag, attrs=attrs, children=children): - return TElement(tag=tag, attrs=attrs, children=tuple(children)) + tnode = TElement(tag=tag, attrs=attrs, children=tuple(children)) + self.tmap[tnode] = open_tag + return tnode case OpenTFragment(children=children): - return TFragment(children=tuple(children)) + tnode = TFragment(children=tuple(children)) + self.tmap[tnode] = open_tag + return tnode case OpenTComponent( start_i_index=start_i_index, children_start_s_index=children_start_s_index, @@ -320,12 +329,14 @@ def finalize_tag( offset_into_children_start_s=offset_into_children_start_s, template=self.get_source().template, ) - return TComponent( + tnode = TComponent( start_i_index=start_i_index, end_i_index=endtag_i_index, children_ref=children_ref, attrs=attrs, ) + self.tmap[tnode] = open_tag + return tnode def extract_component_children_ref( self, @@ -532,6 +543,7 @@ def reset(self): self.stack = [] self.placeholders = PlaceholderState() self.source = None + self.tmap = {} def close(self) -> None: if self.waiting_for_data(): From d0decf4294f36ab8d831eb8d69883930fb94e0e5 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Mon, 15 Jun 2026 10:04:34 -0700 Subject: [PATCH 12/78] Be more surgical about catching nested component error. --- tdom/parser.py | 88 +++++++++++++++++++++++++++++++++------------ tdom/parser_test.py | 39 ++++++++++++++++++-- 2 files changed, 101 insertions(+), 26 deletions(-) diff --git a/tdom/parser.py b/tdom/parser.py index 9e36750a..fcb5770f 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -424,20 +424,6 @@ def validate_end_tag(self, tag: str, open_tag: OpenTag) -> int | None: raise ValueError( "Component end tags must have exactly one interpolation." ) - if not source.expressions_match( - open_tag.start_i_index, tag_ref.i_indexes[0] - ) and not source.values_match( - open_tag.start_i_index, tag_ref.i_indexes[0] - ): - e = TypeError( - "Component start and end tags must contain the same callable." - ) - if self.has_ambiguous_forward_slash(open_tag): - starttag = source.format_starttag(start_i_index) - e.add_note( - f'Did you mean to quote the last attribute or put a space before "/>" for "<{{{starttag}}} .../>"?' - ) - raise e return tag_ref.i_indexes[0] def get_starttag_text(self, msg: str = "Expecting starttag text to be set.") -> str: @@ -498,13 +484,48 @@ def handle_startendtag(self, tag: str, attrs: Sequence[HTMLAttribute]) -> None: def handle_endtag(self, tag: str) -> None: if not self.stack: - raise ValueError(f"Unexpected closing tag with no open tag.") - + tag_ref = self.placeholders.copy().remove_placeholders(tag) + if tag_ref.is_literal: + raise ValueError(f"Unexpected closing tag with no open tag.") + if not tag_ref.is_singleton: + # @TODO: Also it doesn't match anything + raise ValueError( + "Component end tags must have exactly one interpolation." + ) + # Component tag endtag but no component tag is open... + source = self.get_source() + unmatched_endtag = source.format_endtag(tag_ref.i_indexes[0]) + raise ValueError( + f"Unexpected closing component tag with no open tag." + ) open_tag = self.stack.pop() endtag_i_index = self.validate_end_tag(tag, open_tag) final_tag = self.finalize_tag(open_tag, endtag_i_index) self.append_child(final_tag) + def get_closed_tcomps(self, root: OpenTag | None) -> list[TComponent]: + """ + Get TComponents that were closed during parsing starting from `root`. + + If `root` is None then use the parser's default `root`. + + TComponents should be returned in the order they were closed in: + from first closed to last closed. + + @NOTE: That the root is an `OpenTag` but its `children` are actually `TNode`s. + """ + if root is None: + root = self.root + tcomps = [] + nodes = list(root.children) + while nodes: + node = nodes.pop() + if isinstance(node, TComponent): + tcomps.append(node) + elif isinstance(node, (TElement, TFragment)): + nodes.extend(node.children) + return tcomps + # ------------------------------------------ # HTMLParser other callbacks # ------------------------------------------ @@ -557,20 +578,41 @@ def close(self) -> None: "Parser expects more data, is the template valid html?" ) if self.stack: + source = self.get_source() e = ValueError("Invalid HTML structure: unclosed tags remain.") - # Check for tags that might have meant to self-close but whose - # unquoted last attribute value consumed a "/", ie.
. + # @TODO: We need to determine which tags this might apply to, + # this only applies to components. parent = self.stack[-1] - # @TODO: We need to determine which tags this might apply to, this only applies to components. if isinstance(parent, OpenTComponent) and self.has_ambiguous_forward_slash( parent ): - starttag = ( - f"{{{self.get_source().format_starttag(parent.start_i_index)}}}" - ) + # CASE: "<{C1} attr={value}/>" -- meant to self-close + # Maybe user meant to self-close? + starttag = source.format_starttag(parent.start_i_index) e.add_note( - f'Did you mean to quote the last attribute or put a space before "/>" for "<{starttag} .../>"?' + f'Did you mean to quote the last attribute or put a space before "/>" for "<{{{starttag}}} .../>"?' ) + else: + # CASE: "<{C2}><{C1} attr=/>" + # Maybe user meant to self-close <{C1} ...>, but closed by leaving <{C2}...> open? + for comp in reversed(self.get_closed_tcomps(parent)): + if ( + comp.end_i_index is not None + and comp.start_i_index != comp.end_i_index + and not source.values_match( + comp.start_i_index, comp.end_i_index + ) + ): + closed_tag = self.tmap[comp] + starttag = source.format_starttag(comp.start_i_index) + endtag = source.format_endtag(comp.end_i_index) + e.add_note( + f"Component start tag, <{{{starttag}}}>, and end tag, , have values that do not match." + ) + if self.has_ambiguous_forward_slash(closed_tag): + e.add_note( + f'Did you mean to quote the last attribute or put a space before "/>" for "<{{{starttag}}} .../>"?' + ) raise e if not self.placeholders.is_empty: raise ValueError("Some placeholders were never resolved.") diff --git a/tdom/parser_test.py b/tdom/parser_test.py index ad923223..ec3f1530 100644 --- a/tdom/parser_test.py +++ b/tdom/parser_test.py @@ -468,6 +468,14 @@ def Component(): _ = TemplateParser.parse(t"<{Component}>") +def test_unmatched_end_component_tag_error(): + def Component(): + pass + + with pytest.raises(ValueError, match="Unexpected closing component tag"): + _ = TemplateParser.parse(t"") + + def test_placeholder_collision_avoidance(): config = make_placeholder_config() # This test is to ensure that our placeholder detection avoids collisions @@ -604,7 +612,19 @@ def test_extract_with_templated_attr_gt_char(self, Component): ) -class TestComponentUnquotedAttrValue: +class TestComponentUnquotedAttrValueWithAmbiguousSlash: + @pytest.fixture + def comp_maker(self): + def maker(suffix=None): + def _Comp(children: Template, title: str) -> Template: + return children + + if suffix is not None: + _Comp.__name__ = f"{_Comp.__name__}__{suffix}" + return _Comp + + return maker + @pytest.fixture def Comp(self): def _Comp(children: Template, title: str) -> Template: @@ -631,6 +651,19 @@ def test_comp_unquoted_attr_value_error_nested_in_el(self, Comp): ): _ = TemplateParser.parse(t"
<{Comp} title=today/>
") - def test_comp_unquoted_attr_value_error_nested_in_comp(self, Comp, Comp2): - with pytest.raises(TypeError, match="Did you mean to quote the last attribute"): + def test_comp_unquoted_attr_value_error_single_nested_in_comp(self, Comp, Comp2): + with pytest.raises( + ValueError, match="Did you mean to quote the last attribute" + ): _ = TemplateParser.parse(t"<{Comp2}><{Comp} title=today/>") + + @pytest.mark.skip() + def test_comp_unquoted_attr_value_error_double_nested_in_comp(self, comp_maker): + Comp1, Comp2, Comp3 = comp_maker("1"), comp_maker("2"), comp_maker("3") + # @TODO: We should warn about this ambig slash. + with pytest.raises( + ValueError, match="Did you meant to quote the last attribute" + ): + _ = TemplateParser.parse( + t"<{Comp2}><{Comp1}><{Comp3} title=today/>" + ) From 1dc72eddc985ccf0d22eaab26d60942805744f11 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Mon, 15 Jun 2026 10:08:02 -0700 Subject: [PATCH 13/78] Clump parse info into class. --- tdom/parser.py | 47 ++++++++++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/tdom/parser.py b/tdom/parser.py index fcb5770f..9a50ff8e 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -26,13 +26,20 @@ @dataclass(frozen=True) -class OpenTElement: +class ParseInfo: + "Track parse info for error reporting." + starttag_text: str - " Entire starttag as parsed, includes placeholders, used for debugging. " + " Entire starttag as parsed, includes placeholders, . " raw_attrs: Sequence[HTMLAttribute] - " Attrs as parsed, includes placeholders, used for debugging. " + " Attrs as parsed, includes placeholders. " startend: bool - " Was parsed as startend tag, ie. , used for debugging. " + " Was parsed as startend tag, ie. . " + + +@dataclass(frozen=True) +class OpenTElement: + parse_info: ParseInfo tag: str attrs: tuple[TAttribute, ...] children: list[TNode] = field(default_factory=list) @@ -45,12 +52,7 @@ class OpenTFragment: @dataclass(frozen=True) class OpenTComponent: - starttag_text: str - " Entire starttag as parsed, includes placeholders, used for debugging. " - raw_attrs: Sequence[HTMLAttribute] - " Attrs as parsed, includes placeholders, used for debugging. " - startend: bool - " Was parsed as startend tag, ie. , used for debugging. " + parse_info: ParseInfo start_i_index: int children_start_s_index: int """The strings index where the component's children template starts.""" @@ -204,9 +206,11 @@ def make_open_tag( if tag_ref.is_literal: return OpenTElement( - starttag_text=self.get_starttag_text(), - raw_attrs=attrs, - startend=startend, + parse_info=ParseInfo( + starttag_text=self.get_starttag_text(), + raw_attrs=attrs, + startend=startend, + ), tag=tag, attrs=self.make_tattrs(attrs), ) @@ -248,9 +252,9 @@ def make_open_tag( ) return OpenTComponent( - starttag_text=starttag_text, - raw_attrs=attrs, - startend=startend, + parse_info=ParseInfo( + starttag_text=starttag_text, raw_attrs=attrs, startend=startend + ), start_i_index=i_index, children_start_s_index=children_start_s_index, offset_into_children_start_s=offset_into_children_start_s, @@ -450,17 +454,18 @@ def has_ambiguous_forward_slash(self, open_tag: OpenTag) -> bool: with "<{Component} title={title} />". """ if isinstance(open_tag, (OpenTElement, OpenTComponent)): + parse_info = open_tag.parse_info return ( # has attributes - len(open_tag.raw_attrs) > 0 + len(parse_info.raw_attrs) > 0 # last attr not bare attribute - and open_tag.raw_attrs[-1][1] is not None + and parse_info.raw_attrs[-1][1] is not None # last char of last attr is "/" - and open_tag.raw_attrs[-1][1][-1] == "/" + and parse_info.raw_attrs[-1][1][-1] == "/" # parsed starttag ends with "/>" - and open_tag.starttag_text.endswith("/>") + and parse_info.starttag_text.endswith("/>") # if parsed as startend then its not ambiguous - and not open_tag.startend + and not parse_info.startend ) return False From 0d7a0a44ee1c8065e1345259d3cabf8688947a0f Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Mon, 15 Jun 2026 10:08:57 -0700 Subject: [PATCH 14/78] Cut this out for now. --- tdom/parser.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tdom/parser.py b/tdom/parser.py index 9a50ff8e..1b893501 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -93,13 +93,6 @@ def _check_indices(self, index1: int, index2: int): f"Interpolation indices exceed bounds: {index1} {index2}: [0...{last_index}]" ) - def expressions_match(self, i_index1: int, i_index2: int) -> bool: - self._check_indices(i_index1, i_index2) - return ( - self.interpolations[i_index1].expression - == self.interpolations[i_index2].expression - ) - def values_match(self, i_index1: int, i_index2: int) -> bool: self._check_indices(i_index1, i_index2) return ( From ae65b27ab5767690cfc5d11f767453a2db6d7dbf Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Mon, 15 Jun 2026 22:27:17 -0700 Subject: [PATCH 15/78] Fold parse info tracking into source tracker. --- tdom/parser.py | 149 ++++++++++++++++++++++++++++++-------------- tdom/parser_test.py | 4 +- 2 files changed, 102 insertions(+), 51 deletions(-) diff --git a/tdom/parser.py b/tdom/parser.py index 1b893501..2d95280b 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -26,8 +26,8 @@ @dataclass(frozen=True) -class ParseInfo: - "Track parse info for error reporting." +class TagOpening: + "Track parse info at the opening of a tag for error reporting." starttag_text: str " Entire starttag as parsed, includes placeholders, . " @@ -38,21 +38,37 @@ class ParseInfo: @dataclass(frozen=True) -class OpenTElement: - parse_info: ParseInfo +class TagClosing: + "Track info available when a tag is closed." + + original_open_tag: OpenTag + " The original open tag that tracked this tag before it was closed. " + + +class OpenTagBase: + """Mixin to make open tags hash by id and equal by id.""" + + def __hash__(self): + return hash(id(self)) + + def __eq__(self, other): + return id(self) == id(other) + + +@dataclass(frozen=True, eq=False) +class OpenTElement(OpenTagBase): tag: str attrs: tuple[TAttribute, ...] children: list[TNode] = field(default_factory=list) -@dataclass(frozen=True) -class OpenTFragment: +@dataclass(frozen=True, eq=False) +class OpenTFragment(OpenTagBase): children: list[TNode] = field(default_factory=list) -@dataclass(frozen=True) -class OpenTComponent: - parse_info: ParseInfo +@dataclass(frozen=True, eq=False) +class OpenTComponent(OpenTagBase): start_i_index: int children_start_s_index: int """The strings index where the component's children template starts.""" @@ -60,15 +76,19 @@ class OpenTComponent: """The offset INTO the starting string where the component's children template starts.""" attrs: tuple[TAttribute, ...] # @NOTE: The `children` are discarded after parsing and are just used to - # track template consistency. If the component is processed and - # returns its children template then that template will be - # re-parsed (or pulled from the cache). + # track template consistency or assist with error reporting. If the + # component is processed and returns its children template then that + # template will be re-parsed (or pulled from the cache). children: list[TNode] = field(default_factory=list) type OpenTag = OpenTElement | OpenTFragment | OpenTComponent +type TNodeContainer = TComponent | TElement | TFragment +" Alias for union of tnodes that have children. " + + @dataclass class SourceTracker: """Tracks source locations within a Template for error reporting.""" @@ -82,19 +102,41 @@ class SourceTracker: i_index: int = -1 # The current interpolation index. s_index: int = -1 # The current string index. + tag_closings: dict[TNodeContainer, TagClosing] = field(default_factory=dict) + + tag_openings: dict[OpenTag, TagOpening] = field(default_factory=dict) + + def record_tag_opening( + self, + open_tag: OpenTag, + starttag_text: str, + raw_attrs: Sequence[HTMLAttribute], + startend: bool, + ) -> TagOpening: + opening = self.tag_openings[open_tag] = TagOpening( + starttag_text=starttag_text, raw_attrs=raw_attrs, startend=startend + ) + return opening + + def record_tag_closing( + self, tnode: TNodeContainer, open_tag: OpenTag + ) -> TagClosing: + closing = self.tag_closings[tnode] = TagClosing(original_open_tag=open_tag) + return closing + + def get_tag_closing(self, tnode: TNodeContainer) -> TagClosing: + """Get available info when `tnode` was created at closing of a tag.""" + return self.tag_closings[tnode] + + def get_tag_opening(self, open_tag: OpenTag) -> TagOpening: + """Get available info when `open_tag` was created at opening of a tag.""" + return self.tag_openings[open_tag] + @property def interpolations(self) -> tuple[Interpolation, ...]: return self.template.interpolations - def _check_indices(self, index1: int, index2: int): - last_index = len(self.interpolations) - 1 - if max(index1, index2) > last_index or min(index1, index2) < 0: - raise ValueError( - f"Interpolation indices exceed bounds: {index1} {index2}: [0...{last_index}]" - ) - def values_match(self, i_index1: int, i_index2: int) -> bool: - self._check_indices(i_index1, i_index2) return ( self.interpolations[i_index1].value == self.interpolations[i_index2].value ) @@ -131,7 +173,6 @@ class TemplateParser(HTMLParser): stack: list[OpenTag] placeholders: PlaceholderState source: SourceTracker | None - tmap: dict[TComponent | TElement | TFragment, OpenTag] " Map from completed tnodes back to their opentag for error reporting. " def __init__(self, *, convert_charrefs: bool = True): @@ -195,18 +236,20 @@ def make_open_tag( self, tag: str, attrs: Sequence[HTMLAttribute], startend: bool = False ) -> OpenTag: """Build an OpenTag from a raw tag and attribute tuples.""" + source = self.get_source() tag_ref = self.placeholders.remove_placeholders(tag) - if tag_ref.is_literal: - return OpenTElement( - parse_info=ParseInfo( - starttag_text=self.get_starttag_text(), - raw_attrs=attrs, - startend=startend, - ), + open_tag = OpenTElement( tag=tag, attrs=self.make_tattrs(attrs), ) + source.record_tag_opening( + open_tag, + starttag_text=self.get_starttag_text(), + raw_attrs=attrs, + startend=startend, + ) + return open_tag if not tag_ref.is_singleton: raise ValueError( @@ -244,15 +287,19 @@ def make_open_tag( starttag_text=starttag_text, ) - return OpenTComponent( - parse_info=ParseInfo( - starttag_text=starttag_text, raw_attrs=attrs, startend=startend - ), + open_tag = OpenTComponent( start_i_index=i_index, children_start_s_index=children_start_s_index, offset_into_children_start_s=offset_into_children_start_s, attrs=tattrs, ) + source.record_tag_opening( + open_tag, + starttag_text=starttag_text, + raw_attrs=attrs, + startend=startend, + ) + return open_tag def compute_offset_into_children_start_s( self, @@ -304,15 +351,12 @@ def finalize_tag( self, open_tag: OpenTag, endtag_i_index: int | None = None ) -> TNode: """Finalize an OpenTag into a TNode.""" + source = self.get_source() match open_tag: case OpenTElement(tag=tag, attrs=attrs, children=children): tnode = TElement(tag=tag, attrs=attrs, children=tuple(children)) - self.tmap[tnode] = open_tag - return tnode case OpenTFragment(children=children): tnode = TFragment(children=tuple(children)) - self.tmap[tnode] = open_tag - return tnode case OpenTComponent( start_i_index=start_i_index, children_start_s_index=children_start_s_index, @@ -324,7 +368,7 @@ def finalize_tag( endtag_i_index=endtag_i_index, children_start_s_index=children_start_s_index, offset_into_children_start_s=offset_into_children_start_s, - template=self.get_source().template, + template=source.template, ) tnode = TComponent( start_i_index=start_i_index, @@ -332,8 +376,8 @@ def finalize_tag( children_ref=children_ref, attrs=attrs, ) - self.tmap[tnode] = open_tag - return tnode + source.record_tag_closing(tnode, open_tag) + return tnode def extract_component_children_ref( self, @@ -447,7 +491,8 @@ def has_ambiguous_forward_slash(self, open_tag: OpenTag) -> bool: with "<{Component} title={title} />". """ if isinstance(open_tag, (OpenTElement, OpenTComponent)): - parse_info = open_tag.parse_info + source = self.get_source() + parse_info = source.get_tag_opening(open_tag) return ( # has attributes len(parse_info.raw_attrs) > 0 @@ -491,8 +536,7 @@ def handle_endtag(self, tag: str) -> None: "Component end tags must have exactly one interpolation." ) # Component tag endtag but no component tag is open... - source = self.get_source() - unmatched_endtag = source.format_endtag(tag_ref.i_indexes[0]) + unmatched_endtag = self.get_source().format_endtag(tag_ref.i_indexes[0]) raise ValueError( f"Unexpected closing component tag with no open tag." ) @@ -501,7 +545,9 @@ def handle_endtag(self, tag: str) -> None: final_tag = self.finalize_tag(open_tag, endtag_i_index) self.append_child(final_tag) - def get_closed_tcomps(self, root: OpenTag | None) -> list[TComponent]: + def get_closed_tcomps( + self, root: OpenTag | None, recurse_component_children: bool = False + ) -> list[TComponent]: """ Get TComponents that were closed during parsing starting from `root`. @@ -520,6 +566,9 @@ def get_closed_tcomps(self, root: OpenTag | None) -> list[TComponent]: node = nodes.pop() if isinstance(node, TComponent): tcomps.append(node) + if recurse_component_children: + tag_closing = self.get_source().get_tag_closing(node) + nodes.extend(tag_closing.original_open_tag.children) elif isinstance(node, (TElement, TFragment)): nodes.extend(node.children) return tcomps @@ -562,7 +611,6 @@ def reset(self): self.stack = [] self.placeholders = PlaceholderState() self.source = None - self.tmap = {} def close(self) -> None: if self.waiting_for_data(): @@ -591,9 +639,12 @@ def close(self) -> None: f'Did you mean to quote the last attribute or put a space before "/>" for "<{{{starttag}}} .../>"?' ) else: - # CASE: "<{C2}><{C1} attr=/>" + # CASE: t"<{C2}><{C1} attr=/>" # Maybe user meant to self-close <{C1} ...>, but closed by leaving <{C2}...> open? - for comp in reversed(self.get_closed_tcomps(parent)): + # CASE: t"<{C3}><{C2}><{C1} attr=/>" + for comp in reversed( + self.get_closed_tcomps(parent, recurse_component_children=True) + ): if ( comp.end_i_index is not None and comp.start_i_index != comp.end_i_index @@ -601,13 +652,15 @@ def close(self) -> None: comp.start_i_index, comp.end_i_index ) ): - closed_tag = self.tmap[comp] + tag_closing_info = source.get_tag_closing(comp) starttag = source.format_starttag(comp.start_i_index) endtag = source.format_endtag(comp.end_i_index) e.add_note( f"Component start tag, <{{{starttag}}}>, and end tag, , have values that do not match." ) - if self.has_ambiguous_forward_slash(closed_tag): + if self.has_ambiguous_forward_slash( + tag_closing_info.original_open_tag + ): e.add_note( f'Did you mean to quote the last attribute or put a space before "/>" for "<{{{starttag}}} .../>"?' ) diff --git a/tdom/parser_test.py b/tdom/parser_test.py index ec3f1530..4a1fa98e 100644 --- a/tdom/parser_test.py +++ b/tdom/parser_test.py @@ -657,12 +657,10 @@ def test_comp_unquoted_attr_value_error_single_nested_in_comp(self, Comp, Comp2) ): _ = TemplateParser.parse(t"<{Comp2}><{Comp} title=today/>") - @pytest.mark.skip() def test_comp_unquoted_attr_value_error_double_nested_in_comp(self, comp_maker): Comp1, Comp2, Comp3 = comp_maker("1"), comp_maker("2"), comp_maker("3") - # @TODO: We should warn about this ambig slash. with pytest.raises( - ValueError, match="Did you meant to quote the last attribute" + ValueError, match="Did you mean to quote the last attribute" ): _ = TemplateParser.parse( t"<{Comp2}><{Comp1}><{Comp3} title=today/>" From 1d7a8ad04d631c125a59d2482aefe3d8c451cab3 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Wed, 24 Jun 2026 12:04:21 -0700 Subject: [PATCH 16/78] Backport more consistent parser position and source info tracking. --- tdom/parser.py | 201 ++++++++++++++++++++++--------------------- tdom/parser_utils.py | 1 + tdom/source.py | 46 ++++++++++ tdom/tnodes.py | 13 +++ 4 files changed, 161 insertions(+), 100 deletions(-) create mode 100644 tdom/parser_utils.py create mode 100644 tdom/source.py diff --git a/tdom/parser.py b/tdom/parser.py index 2d95280b..25656b1c 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -4,7 +4,12 @@ from string.templatelib import Interpolation, Template from .htmlspec import VOID_ELEMENTS +from .parser_utils import HTMLAttribute from .placeholders import PlaceholderConfig, PlaceholderState +from .source import ( + FrozenPosition, + TagSourceInfo, +) from .template_utils import TemplateRef, combine_template_refs from .tnodes import ( TAttribute, @@ -21,60 +26,58 @@ TText, ) -type HTMLAttribute = tuple[str, str | None] -type HTMLAttributesDict = dict[str, str | None] - -@dataclass(frozen=True) -class TagOpening: - "Track parse info at the opening of a tag for error reporting." +@dataclass(frozen=True, slots=True) +class OpenTagSourceInfo: + """ + Retained tag information from the parsed source. + @NOTE: These properties DEPEND on the placeholder configuration because + they can contain embedded placeholders. + """ starttag_text: str " Entire starttag as parsed, includes placeholders, . " - raw_attrs: Sequence[HTMLAttribute] + raw_attrs: tuple[HTMLAttribute, ...] " Attrs as parsed, includes placeholders. " startend: bool " Was parsed as startend tag, ie. . " + starttag_pos: FrozenPosition + " Position of the parser when the element starttag was parsed. " + def close(self, endtag_pos: FrozenPosition | None = None) -> TagSourceInfo: + return TagSourceInfo( + starttag_text=self.starttag_text, + raw_attrs=self.raw_attrs, + startend=self.startend, + starttag_pos=self.starttag_pos, + endtag_pos=endtag_pos) -@dataclass(frozen=True) -class TagClosing: - "Track info available when a tag is closed." - - original_open_tag: OpenTag - " The original open tag that tracked this tag before it was closed. " - - -class OpenTagBase: - """Mixin to make open tags hash by id and equal by id.""" - def __hash__(self): - return hash(id(self)) - - def __eq__(self, other): - return id(self) == id(other) - - -@dataclass(frozen=True, eq=False) -class OpenTElement(OpenTagBase): +@dataclass +class OpenTElement: tag: str attrs: tuple[TAttribute, ...] + parser_pos: FrozenPosition + sinfo: OpenTagSourceInfo children: list[TNode] = field(default_factory=list) -@dataclass(frozen=True, eq=False) -class OpenTFragment(OpenTagBase): +@dataclass +class OpenTFragment: + parser_pos: FrozenPosition | None = None children: list[TNode] = field(default_factory=list) -@dataclass(frozen=True, eq=False) -class OpenTComponent(OpenTagBase): +@dataclass +class OpenTComponent: start_i_index: int children_start_s_index: int """The strings index where the component's children template starts.""" offset_into_children_start_s: int """The offset INTO the starting string where the component's children template starts.""" attrs: tuple[TAttribute, ...] + parser_pos: FrozenPosition + sinfo: OpenTagSourceInfo # @NOTE: The `children` are discarded after parsing and are just used to # track template consistency or assist with error reporting. If the # component is processed and returns its children template then that @@ -85,10 +88,6 @@ class OpenTComponent(OpenTagBase): type OpenTag = OpenTElement | OpenTFragment | OpenTComponent -type TNodeContainer = TComponent | TElement | TFragment -" Alias for union of tnodes that have children. " - - @dataclass class SourceTracker: """Tracks source locations within a Template for error reporting.""" @@ -102,36 +101,6 @@ class SourceTracker: i_index: int = -1 # The current interpolation index. s_index: int = -1 # The current string index. - tag_closings: dict[TNodeContainer, TagClosing] = field(default_factory=dict) - - tag_openings: dict[OpenTag, TagOpening] = field(default_factory=dict) - - def record_tag_opening( - self, - open_tag: OpenTag, - starttag_text: str, - raw_attrs: Sequence[HTMLAttribute], - startend: bool, - ) -> TagOpening: - opening = self.tag_openings[open_tag] = TagOpening( - starttag_text=starttag_text, raw_attrs=raw_attrs, startend=startend - ) - return opening - - def record_tag_closing( - self, tnode: TNodeContainer, open_tag: OpenTag - ) -> TagClosing: - closing = self.tag_closings[tnode] = TagClosing(original_open_tag=open_tag) - return closing - - def get_tag_closing(self, tnode: TNodeContainer) -> TagClosing: - """Get available info when `tnode` was created at closing of a tag.""" - return self.tag_closings[tnode] - - def get_tag_opening(self, open_tag: OpenTag) -> TagOpening: - """Get available info when `open_tag` was created at opening of a tag.""" - return self.tag_openings[open_tag] - @property def interpolations(self) -> tuple[Interpolation, ...]: return self.template.interpolations @@ -174,6 +143,11 @@ class TemplateParser(HTMLParser): placeholders: PlaceholderState source: SourceTracker | None " Map from completed tnodes back to their opentag for error reporting. " + tcomponent_children: dict[TComponent, list[TNode]] + "List of children for each finished tcomponent, stored at closing. " + sinfo_table: dict[FrozenPosition, TagSourceInfo] + " Tags with more source info than just a position are tracked in this mapping. " + def __init__(self, *, convert_charrefs: bool = True): # This calls HTMLParser.reset() which we override to set up our state. @@ -191,6 +165,18 @@ def append_child(self, child: TNode) -> None: parent = self.get_parent() parent.children.append(child) + def get_parser_pos(self) -> FrozenPosition: + """ + Get the current position of the parser. + + @NOTE: This position is relative to text embedded with placeholders but + can be translated back to the position within the original template. + Since it *IS* relative to placeholders, ie. "SLOTS", this position is + unique across a "family" of templates with the same structure. + """ + line, offset = self.getpos() + return FrozenPosition(line=line, offset=offset) + # ------------------------------------------ # Attribute Helpers # ------------------------------------------ @@ -236,18 +222,19 @@ def make_open_tag( self, tag: str, attrs: Sequence[HTMLAttribute], startend: bool = False ) -> OpenTag: """Build an OpenTag from a raw tag and attribute tuples.""" - source = self.get_source() tag_ref = self.placeholders.remove_placeholders(tag) if tag_ref.is_literal: + parser_pos = self.get_parser_pos() open_tag = OpenTElement( tag=tag, attrs=self.make_tattrs(attrs), - ) - source.record_tag_opening( - open_tag, - starttag_text=self.get_starttag_text(), - raw_attrs=attrs, - startend=startend, + sinfo=OpenTagSourceInfo( + starttag_text=self.get_starttag_text(), + raw_attrs=tuple(attrs), + startend=startend, + starttag_pos=parser_pos, + ), + parser_pos=parser_pos, ) return open_tag @@ -287,18 +274,19 @@ def make_open_tag( starttag_text=starttag_text, ) + parser_pos = self.get_parser_pos() open_tag = OpenTComponent( start_i_index=i_index, children_start_s_index=children_start_s_index, offset_into_children_start_s=offset_into_children_start_s, attrs=tattrs, - ) - source.record_tag_opening( - open_tag, - starttag_text=starttag_text, - raw_attrs=attrs, - startend=startend, - ) + parser_pos=parser_pos, + sinfo=OpenTagSourceInfo( + starttag_text=starttag_text, + raw_attrs=tuple(attrs), + startend=startend, + starttag_pos=parser_pos + )) return open_tag def compute_offset_into_children_start_s( @@ -348,20 +336,30 @@ def compute_offset_into_children_start_s( return len(tag_ref.strings[-1]) def finalize_tag( - self, open_tag: OpenTag, endtag_i_index: int | None = None + self, open_tag: OpenTag, endtag_i_index: int | None = None, endtag_pos: FrozenPosition | None = None ) -> TNode: """Finalize an OpenTag into a TNode.""" source = self.get_source() match open_tag: - case OpenTElement(tag=tag, attrs=attrs, children=children): - tnode = TElement(tag=tag, attrs=attrs, children=tuple(children)) - case OpenTFragment(children=children): - tnode = TFragment(children=tuple(children)) + case OpenTElement( + tag=tag, + attrs=attrs, + children=children, + parser_pos=parser_pos, + sinfo=sinfo + ): + tnode = TElement(tag=tag, attrs=attrs, children=tuple(children), parser_pos=parser_pos) + self.sinfo_table[parser_pos] = sinfo.close(endtag_pos=endtag_pos) + case OpenTFragment(children=children, parser_pos=parser_pos): + tnode = TFragment(children=tuple(children), parser_pos=parser_pos) case OpenTComponent( start_i_index=start_i_index, children_start_s_index=children_start_s_index, offset_into_children_start_s=offset_into_children_start_s, attrs=attrs, + parser_pos=parser_pos, + sinfo=sinfo, + children=children, ): children_ref = self.extract_component_children_ref( start_i_index=start_i_index, @@ -375,8 +373,10 @@ def finalize_tag( end_i_index=endtag_i_index, children_ref=children_ref, attrs=attrs, + parser_pos=parser_pos ) - source.record_tag_closing(tnode, open_tag) + self.sinfo_table[parser_pos] = sinfo.close(endtag_pos=endtag_pos) + self.tcomponent_children[tnode] = children return tnode def extract_component_children_ref( @@ -456,7 +456,7 @@ def validate_end_tag(self, tag: str, open_tag: OpenTag) -> int | None: e = ValueError( f"Mismatched closing tag for component with tag {{{starttag}}}." ) - if self.has_ambiguous_forward_slash(open_tag): + if self.has_ambiguous_forward_slash(open_tag.sinfo): e.add_note( f'Did you mean to quote the last attribute or put a space before "/>" for "<{{{starttag}}} .../>"?' ) @@ -478,7 +478,7 @@ def get_starttag_text(self, msg: str = "Expecting starttag text to be set.") -> raise AssertionError(msg) return starttag_text - def has_ambiguous_forward_slash(self, open_tag: OpenTag) -> bool: + def has_ambiguous_forward_slash(self, sinfo: OpenTagSourceInfo | TagSourceInfo | None) -> bool: """ Detect when an unquoted attribute value consumes a trailing "/" that *might* have been meant to attempt to self-close a tag, ie. "/>". @@ -490,20 +490,18 @@ def has_ambiguous_forward_slash(self, open_tag: OpenTag) -> bool: Or more often "<{Component} title={title}/>" which should be corrected with "<{Component} title={title} />". """ - if isinstance(open_tag, (OpenTElement, OpenTComponent)): - source = self.get_source() - parse_info = source.get_tag_opening(open_tag) + if sinfo is not None: return ( # has attributes - len(parse_info.raw_attrs) > 0 + len(sinfo.raw_attrs) > 0 # last attr not bare attribute - and parse_info.raw_attrs[-1][1] is not None + and sinfo.raw_attrs[-1][1] is not None # last char of last attr is "/" - and parse_info.raw_attrs[-1][1][-1] == "/" + and sinfo.raw_attrs[-1][1][-1] == "/" # parsed starttag ends with "/>" - and parse_info.starttag_text.endswith("/>") + and sinfo.starttag_text.endswith("/>") # if parsed as startend then its not ambiguous - and not parse_info.startend + and not sinfo.startend ) return False @@ -542,7 +540,8 @@ def handle_endtag(self, tag: str) -> None: ) open_tag = self.stack.pop() endtag_i_index = self.validate_end_tag(tag, open_tag) - final_tag = self.finalize_tag(open_tag, endtag_i_index) + final_tag = self.finalize_tag( + open_tag, endtag_i_index=endtag_i_index, endtag_pos=self.get_parser_pos()) self.append_child(final_tag) def get_closed_tcomps( @@ -567,8 +566,8 @@ def get_closed_tcomps( if isinstance(node, TComponent): tcomps.append(node) if recurse_component_children: - tag_closing = self.get_source().get_tag_closing(node) - nodes.extend(tag_closing.original_open_tag.children) + children = self.tcomponent_children.get(node, []) + nodes.extend(children) elif isinstance(node, (TElement, TFragment)): nodes.extend(node.children) return tcomps @@ -611,6 +610,8 @@ def reset(self): self.stack = [] self.placeholders = PlaceholderState() self.source = None + self.sinfo_table = {} + self.tcomponent_children = {} def close(self) -> None: if self.waiting_for_data(): @@ -630,7 +631,7 @@ def close(self) -> None: # this only applies to components. parent = self.stack[-1] if isinstance(parent, OpenTComponent) and self.has_ambiguous_forward_slash( - parent + parent.sinfo ): # CASE: "<{C1} attr={value}/>" -- meant to self-close # Maybe user meant to self-close? @@ -652,14 +653,14 @@ def close(self) -> None: comp.start_i_index, comp.end_i_index ) ): - tag_closing_info = source.get_tag_closing(comp) + sinfo = self.sinfo_table.get(comp.parser_pos) if comp.parser_pos is not None else None starttag = source.format_starttag(comp.start_i_index) endtag = source.format_endtag(comp.end_i_index) e.add_note( f"Component start tag, <{{{starttag}}}>, and end tag, , have values that do not match." ) if self.has_ambiguous_forward_slash( - tag_closing_info.original_open_tag + sinfo ): e.add_note( f'Did you mean to quote the last attribute or put a space before "/>" for "<{{{starttag}}} .../>"?' diff --git a/tdom/parser_utils.py b/tdom/parser_utils.py new file mode 100644 index 00000000..f8365871 --- /dev/null +++ b/tdom/parser_utils.py @@ -0,0 +1 @@ +type HTMLAttribute = tuple[str, str | None] diff --git a/tdom/source.py b/tdom/source.py new file mode 100644 index 00000000..3a99e233 --- /dev/null +++ b/tdom/source.py @@ -0,0 +1,46 @@ +from dataclasses import dataclass + +from .parser_utils import HTMLAttribute + + +@dataclass(slots=True, frozen=True) +class FrozenPosition: + "A immutable position in a block of source code." + + line: int = 1 + " Line of code, starts at 1. " + offset: int = 0 + " Offset from the start of the line, starts at 0. " + + +@dataclass(slots=True) +class Position: + "A position in a block of source code." + + line: int = 1 + " Line of code, starts at 1. " + offset: int = 0 + " Offset from the start of the line, starts at 0. " + + def freeze(self) -> FrozenPosition: + return FrozenPosition(line=self.line, offset=self.offset) + + +@dataclass(frozen=True, slots=True) +class TagSourceInfo: + """ + Retained tag information from the parsed source. + + @NOTE: These properties DEPEND on the placeholder configuration because + they can contain embedded placeholders. + """ + starttag_text: str + " Entire starttag as parsed, includes placeholders, . " + raw_attrs: tuple[HTMLAttribute, ...] + " Attrs as parsed, includes placeholders. " + startend: bool + " Was parsed as startend tag, ie. . " + starttag_pos: FrozenPosition + " Position of the parser when the element starttag was parsed. " + endtag_pos: FrozenPosition | None = None + " Position of the parser when the element endtag was parsed. " diff --git a/tdom/tnodes.py b/tdom/tnodes.py index 3afb1063..cee6d6da 100644 --- a/tdom/tnodes.py +++ b/tdom/tnodes.py @@ -1,6 +1,7 @@ import typing as t from dataclasses import dataclass, field +from .source import FrozenPosition from .template_utils import TemplateRef @@ -45,6 +46,8 @@ def __str__(self) -> str: class TText(TNode): ref: TemplateRef + parser_pos: FrozenPosition | None = field(default=None, compare=False) + @classmethod def empty(cls) -> t.Self: return cls(TemplateRef.empty()) @@ -58,6 +61,8 @@ def literal(cls, text: str) -> t.Self: class TComment(TNode): ref: TemplateRef + parser_pos: FrozenPosition | None = field(default=None, compare=False) + @classmethod def literal(cls, text: str) -> t.Self: return cls(TemplateRef.literal(text)) @@ -67,11 +72,15 @@ def literal(cls, text: str) -> t.Self: class TDocumentType(TNode): text: str + parser_pos: FrozenPosition | None = field(default=None, compare=False) + @dataclass(slots=True, frozen=True) class TFragment(TNode): children: tuple[TNode, ...] = field(default_factory=tuple) + parser_pos: FrozenPosition | None = field(default=None, compare=False) + @dataclass(slots=True, frozen=True) class TElement(TNode): @@ -79,6 +88,8 @@ class TElement(TNode): attrs: tuple[TAttribute, ...] = field(default_factory=tuple) children: tuple[TNode, ...] = field(default_factory=tuple) + parser_pos: FrozenPosition | None = field(default=None, compare=False) + @dataclass(slots=True, frozen=True) class TComponent(TNode): @@ -95,5 +106,7 @@ class TComponent(TNode): attrs: tuple[TAttribute, ...] = field(default_factory=tuple) + parser_pos: FrozenPosition | None = field(default=None, compare=False) + type TTag = TElement | TComponent | TFragment From e47303c751d91f031c527f22c62084c5b33bc2e2 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Wed, 24 Jun 2026 12:09:20 -0700 Subject: [PATCH 17/78] Formatting. --- tdom/parser.py | 43 +++++++++++++++++++++++++++++-------------- tdom/source.py | 1 + 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/tdom/parser.py b/tdom/parser.py index 25656b1c..58218f96 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -35,6 +35,7 @@ class OpenTagSourceInfo: @NOTE: These properties DEPEND on the placeholder configuration because they can contain embedded placeholders. """ + starttag_text: str " Entire starttag as parsed, includes placeholders, . " raw_attrs: tuple[HTMLAttribute, ...] @@ -50,7 +51,8 @@ def close(self, endtag_pos: FrozenPosition | None = None) -> TagSourceInfo: raw_attrs=self.raw_attrs, startend=self.startend, starttag_pos=self.starttag_pos, - endtag_pos=endtag_pos) + endtag_pos=endtag_pos, + ) @dataclass @@ -148,7 +150,6 @@ class TemplateParser(HTMLParser): sinfo_table: dict[FrozenPosition, TagSourceInfo] " Tags with more source info than just a position are tracked in this mapping. " - def __init__(self, *, convert_charrefs: bool = True): # This calls HTMLParser.reset() which we override to set up our state. super().__init__(convert_charrefs=convert_charrefs) @@ -285,8 +286,9 @@ def make_open_tag( starttag_text=starttag_text, raw_attrs=tuple(attrs), startend=startend, - starttag_pos=parser_pos - )) + starttag_pos=parser_pos, + ), + ) return open_tag def compute_offset_into_children_start_s( @@ -336,7 +338,10 @@ def compute_offset_into_children_start_s( return len(tag_ref.strings[-1]) def finalize_tag( - self, open_tag: OpenTag, endtag_i_index: int | None = None, endtag_pos: FrozenPosition | None = None + self, + open_tag: OpenTag, + endtag_i_index: int | None = None, + endtag_pos: FrozenPosition | None = None, ) -> TNode: """Finalize an OpenTag into a TNode.""" source = self.get_source() @@ -346,9 +351,14 @@ def finalize_tag( attrs=attrs, children=children, parser_pos=parser_pos, - sinfo=sinfo + sinfo=sinfo, ): - tnode = TElement(tag=tag, attrs=attrs, children=tuple(children), parser_pos=parser_pos) + tnode = TElement( + tag=tag, + attrs=attrs, + children=tuple(children), + parser_pos=parser_pos, + ) self.sinfo_table[parser_pos] = sinfo.close(endtag_pos=endtag_pos) case OpenTFragment(children=children, parser_pos=parser_pos): tnode = TFragment(children=tuple(children), parser_pos=parser_pos) @@ -373,7 +383,7 @@ def finalize_tag( end_i_index=endtag_i_index, children_ref=children_ref, attrs=attrs, - parser_pos=parser_pos + parser_pos=parser_pos, ) self.sinfo_table[parser_pos] = sinfo.close(endtag_pos=endtag_pos) self.tcomponent_children[tnode] = children @@ -478,7 +488,9 @@ def get_starttag_text(self, msg: str = "Expecting starttag text to be set.") -> raise AssertionError(msg) return starttag_text - def has_ambiguous_forward_slash(self, sinfo: OpenTagSourceInfo | TagSourceInfo | None) -> bool: + def has_ambiguous_forward_slash( + self, sinfo: OpenTagSourceInfo | TagSourceInfo | None + ) -> bool: """ Detect when an unquoted attribute value consumes a trailing "/" that *might* have been meant to attempt to self-close a tag, ie. "/>". @@ -541,7 +553,8 @@ def handle_endtag(self, tag: str) -> None: open_tag = self.stack.pop() endtag_i_index = self.validate_end_tag(tag, open_tag) final_tag = self.finalize_tag( - open_tag, endtag_i_index=endtag_i_index, endtag_pos=self.get_parser_pos()) + open_tag, endtag_i_index=endtag_i_index, endtag_pos=self.get_parser_pos() + ) self.append_child(final_tag) def get_closed_tcomps( @@ -653,15 +666,17 @@ def close(self) -> None: comp.start_i_index, comp.end_i_index ) ): - sinfo = self.sinfo_table.get(comp.parser_pos) if comp.parser_pos is not None else None + sinfo = ( + self.sinfo_table.get(comp.parser_pos) + if comp.parser_pos is not None + else None + ) starttag = source.format_starttag(comp.start_i_index) endtag = source.format_endtag(comp.end_i_index) e.add_note( f"Component start tag, <{{{starttag}}}>, and end tag, , have values that do not match." ) - if self.has_ambiguous_forward_slash( - sinfo - ): + if self.has_ambiguous_forward_slash(sinfo): e.add_note( f'Did you mean to quote the last attribute or put a space before "/>" for "<{{{starttag}}} .../>"?' ) diff --git a/tdom/source.py b/tdom/source.py index 3a99e233..7ff90726 100644 --- a/tdom/source.py +++ b/tdom/source.py @@ -34,6 +34,7 @@ class TagSourceInfo: @NOTE: These properties DEPEND on the placeholder configuration because they can contain embedded placeholders. """ + starttag_text: str " Entire starttag as parsed, includes placeholders, . " raw_attrs: tuple[HTMLAttribute, ...] From 89b052a15f32306ea53dff54324044d19980a0ce Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Wed, 24 Jun 2026 12:36:12 -0700 Subject: [PATCH 18/78] Actually store positions on other elements. --- tdom/parser.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tdom/parser.py b/tdom/parser.py index 58218f96..92fab52d 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -593,15 +593,18 @@ def handle_data(self, data: str) -> None: ref = self.placeholders.remove_placeholders(data) parent = self.get_parent() if parent.children and isinstance(parent.children[-1], TText): + prior_text = parent.children[-1] parent.children[-1] = TText( - ref=combine_template_refs(parent.children[-1].ref, ref) + ref=combine_template_refs(prior_text.ref, ref), + # Keep starting position of the prior text + parser_pos=prior_text.parser_pos, ) else: - self.append_child(TText(ref=ref)) + self.append_child(TText(ref=ref, parser_pos=self.get_parser_pos())) def handle_comment(self, data: str) -> None: ref = self.placeholders.remove_placeholders(data) - comment = TComment(ref) + comment = TComment(ref, parser_pos=self.get_parser_pos()) self.append_child(comment) def handle_decl(self, decl: str) -> None: @@ -610,7 +613,7 @@ def handle_decl(self, decl: str) -> None: raise ValueError("Interpolations are not allowed in declarations.") elif decl.upper().startswith("DOCTYPE "): doctype_content = decl[7:].strip() - doctype = TDocumentType(doctype_content) + doctype = TDocumentType(doctype_content, parser_pos=self.get_parser_pos()) self.append_child(doctype) else: raise NotImplementedError( From 4f5077add09c11fd65988cb89f46184d6ccc245d Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Wed, 24 Jun 2026 12:37:20 -0700 Subject: [PATCH 19/78] Test that the parser position is actually being set on the nodes. --- tdom/parser_test.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tdom/parser_test.py b/tdom/parser_test.py index 4a1fa98e..bda49914 100644 --- a/tdom/parser_test.py +++ b/tdom/parser_test.py @@ -4,6 +4,7 @@ from .parser import TemplateParser from .placeholders import make_placeholder_config +from .source import FrozenPosition from .template_utils import TemplateRef from .tnodes import ( TComment, @@ -665,3 +666,25 @@ def test_comp_unquoted_attr_value_error_double_nested_in_comp(self, comp_maker): _ = TemplateParser.parse( t"<{Comp2}><{Comp1}><{Comp3} title=today/>" ) + + +def PositionComp() -> Template: + return t"" + + +@pytest.mark.parametrize( + "chunk", + ( + (TElement, t""), + (TComment, t""), + (TDocumentType, t""), + (TComponent, t"<{PositionComp}>"), + (TText, t"Just a simple text."), + ), +) +def test_tnode_parser_position(chunk): + tnode = TemplateParser.parse(t"
" + chunk[1] + t"
") + assert tnode.tag == "div" and len(tnode.children) == 1 + el = tnode.children[0] + assert isinstance(el, chunk[0]) + assert el.parser_pos == FrozenPosition(line=1, offset=len("
")) From 2abcb856ec7a11a39a82e44cdf87dbc6ec8484e4 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Wed, 24 Jun 2026 12:46:33 -0700 Subject: [PATCH 20/78] Fold loop into test as tempfix for type issues around parser_pos. --- tdom/parser_test.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/tdom/parser_test.py b/tdom/parser_test.py index bda49914..523b918f 100644 --- a/tdom/parser_test.py +++ b/tdom/parser_test.py @@ -672,19 +672,20 @@ def PositionComp() -> Template: return t"" -@pytest.mark.parametrize( - "chunk", - ( +def test_tnode_parser_position(): + for tnode_type, fragment in ( (TElement, t""), (TComment, t""), (TDocumentType, t""), (TComponent, t"<{PositionComp}>"), (TText, t"Just a simple text."), - ), -) -def test_tnode_parser_position(chunk): - tnode = TemplateParser.parse(t"
" + chunk[1] + t"
") - assert tnode.tag == "div" and len(tnode.children) == 1 - el = tnode.children[0] - assert isinstance(el, chunk[0]) - assert el.parser_pos == FrozenPosition(line=1, offset=len("
")) + ): + tnode = TemplateParser.parse(t"
" + fragment + t"
") + assert ( + isinstance(tnode, TElement) + and tnode.tag == "div" + and len(tnode.children) == 1 + ) + el = tnode.children[0] + assert isinstance(el, tnode_type) + assert el.parser_pos == FrozenPosition(line=1, offset=len("
")) From d462bd49def0f782e566327cd29268e37055b18e Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Wed, 24 Jun 2026 13:52:36 -0700 Subject: [PATCH 21/78] Coerce SourceTracker into Iterator and consolidate placeholder tracking there. --- tdom/parser.py | 100 ++++++++++++++++++++++++++++--------------------- 1 file changed, 57 insertions(+), 43 deletions(-) diff --git a/tdom/parser.py b/tdom/parser.py index 92fab52d..be623987 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -5,7 +5,11 @@ from .htmlspec import VOID_ELEMENTS from .parser_utils import HTMLAttribute -from .placeholders import PlaceholderConfig, PlaceholderState +from .placeholders import ( + PlaceholderConfig, + PlaceholderState, + make_placeholder_config, +) from .source import ( FrozenPosition, TagSourceInfo, @@ -98,11 +102,32 @@ class SourceTracker: # template itself in context and the relevant line/column underlined/etc. template: Template + + placeholders: PlaceholderState + # if i_index >= s_index, feeding an interpolation; # otherwise, when i_index < s_index, feeding a string. i_index: int = -1 # The current interpolation index. s_index: int = -1 # The current string index. + def __iter__(self): + return self + + def __next__(self): + if self.i_index < self.s_index: + # Advance into the next interpolation UNLESS the last string + # we returned was at the end of the template. + if self.s_index == len(self.template.strings) - 1: + raise StopIteration + self.i_index += 1 + return self.placeholders.add_placeholder(self.i_index) + elif self.i_index == self.s_index: + # Advance into the next string + self.s_index += 1 + return self.template.strings[self.s_index] + else: + raise AssertionError("{self.i_index=} should not exceed {self.s_index=}") + @property def interpolations(self) -> tuple[Interpolation, ...]: return self.template.interpolations @@ -112,15 +137,6 @@ def values_match(self, i_index1: int, i_index2: int) -> bool: self.interpolations[i_index1].value == self.interpolations[i_index2].value ) - def advance_interpolation(self) -> int: - """Call before processing an interpolation to move to the next one.""" - self.i_index += 1 - return self.i_index - - def advance_string(self) -> int: - self.s_index += 1 - return self.s_index - def get_expression( self, i_index: int, fallback_prefix: str = "interpolation" ) -> str: @@ -142,7 +158,6 @@ def format_endtag(self, i_index: int) -> str: class TemplateParser(HTMLParser): root: OpenTFragment stack: list[OpenTag] - placeholders: PlaceholderState source: SourceTracker | None " Map from completed tnodes back to their opentag for error reporting. " tcomponent_children: dict[TComponent, list[TNode]] @@ -186,10 +201,12 @@ def make_tattr(self, attr: HTMLAttribute) -> TAttribute: """Build a TAttribute from a raw attribute tuple.""" name, value = attr - - name_ref = self.placeholders.remove_placeholders(name) + source = self.get_source() + name_ref = source.placeholders.remove_placeholders(name) value_ref = ( - self.placeholders.remove_placeholders(value) if value is not None else None + source.placeholders.remove_placeholders(value) + if value is not None + else None ) if name_ref.is_literal: @@ -223,7 +240,8 @@ def make_open_tag( self, tag: str, attrs: Sequence[HTMLAttribute], startend: bool = False ) -> OpenTag: """Build an OpenTag from a raw tag and attribute tuples.""" - tag_ref = self.placeholders.remove_placeholders(tag) + source = self.get_source() + tag_ref = source.placeholders.remove_placeholders(tag) if tag_ref.is_literal: parser_pos = self.get_parser_pos() open_tag = OpenTElement( @@ -271,7 +289,7 @@ def make_open_tag( offset_into_children_start_s = self.compute_offset_into_children_start_s( start_i_index=i_index, tattrs=tattrs, - config=self.placeholders.config, + config=source.placeholders.config, starttag_text=starttag_text, ) @@ -443,7 +461,7 @@ def extract_component_children_ref( def validate_end_tag(self, tag: str, open_tag: OpenTag) -> int | None: """Validate that closing tag matches open tag. Return component end index if applicable.""" source = self.get_source() - tag_ref = self.placeholders.remove_placeholders(tag) + tag_ref = source.placeholders.remove_placeholders(tag) match open_tag: case OpenTElement(): @@ -537,7 +555,8 @@ def handle_startendtag(self, tag: str, attrs: Sequence[HTMLAttribute]) -> None: def handle_endtag(self, tag: str) -> None: if not self.stack: - tag_ref = self.placeholders.copy().remove_placeholders(tag) + source = self.get_source() + tag_ref = source.placeholders.copy().remove_placeholders(tag) if tag_ref.is_literal: raise ValueError(f"Unexpected closing tag with no open tag.") if not tag_ref.is_singleton: @@ -590,7 +609,8 @@ def get_closed_tcomps( # ------------------------------------------ def handle_data(self, data: str) -> None: - ref = self.placeholders.remove_placeholders(data) + source = self.get_source() + ref = source.placeholders.remove_placeholders(data) parent = self.get_parent() if parent.children and isinstance(parent.children[-1], TText): prior_text = parent.children[-1] @@ -603,12 +623,14 @@ def handle_data(self, data: str) -> None: self.append_child(TText(ref=ref, parser_pos=self.get_parser_pos())) def handle_comment(self, data: str) -> None: - ref = self.placeholders.remove_placeholders(data) + source = self.get_source() + ref = source.placeholders.remove_placeholders(data) comment = TComment(ref, parser_pos=self.get_parser_pos()) self.append_child(comment) def handle_decl(self, decl: str) -> None: - ref = self.placeholders.remove_placeholders(decl) + source = self.get_source() + ref = source.placeholders.remove_placeholders(decl) if not ref.is_literal: raise ValueError("Interpolations are not allowed in declarations.") elif decl.upper().startswith("DOCTYPE "): @@ -624,12 +646,12 @@ def reset(self): super().reset() self.root = OpenTFragment() self.stack = [] - self.placeholders = PlaceholderState() self.source = None self.sinfo_table = {} self.tcomponent_children = {} def close(self) -> None: + source = self.get_source() if self.waiting_for_data(): # We apply heuristics here to try to guess why the parser didn't finish. if self.rawdata.count('"') % 2 == 1 or self.rawdata.count("'") % 2 == 1: @@ -641,7 +663,6 @@ def close(self) -> None: "Parser expects more data, is the template valid html?" ) if self.stack: - source = self.get_source() e = ValueError("Invalid HTML structure: unclosed tags remain.") # @TODO: We need to determine which tags this might apply to, # this only applies to components. @@ -684,7 +705,7 @@ def close(self) -> None: f'Did you mean to quote the last attribute or put a space before "/>" for "<{{{starttag}}} .../>"?' ) raise e - if not self.placeholders.is_empty: + if not source.placeholders.is_empty: raise ValueError("Some placeholders were never resolved.") super().close() @@ -721,34 +742,27 @@ def get_source(self) -> SourceTracker: raise AssertionError("Source has not been initialized.") return self.source - def feed_str(self, s: str) -> None: - """Feed a string part of a Template to the parser.""" - self.feed(s) - - def feed_interpolation(self, index: int) -> None: - placeholder = self.placeholders.add_placeholder(index) - self.feed(placeholder) - - def feed_template(self, template: Template) -> None: + def feed_template( + self, template: Template, placeholder_config: PlaceholderConfig + ) -> None: """Feed a Template's content to the parser.""" assert self.source is None, "Did you forget to call reset?" - self.source = SourceTracker(template) - for i_index in range(len(template.interpolations)): - self.source.advance_string() - self.feed_str(template.strings[i_index]) - self.source.advance_interpolation() - self.feed_interpolation(i_index) - self.source.advance_string() - self.feed_str(template.strings[-1]) + self.source = SourceTracker( + template, placeholders=PlaceholderState(config=placeholder_config) + ) + for content in self.source: + self.feed(content) @staticmethod - def parse(t: Template) -> TNode: + def parse(t: Template, config: PlaceholderConfig | None = None) -> TNode: """ Parse a Template containing valid HTML and substitutions and return a TNode tree representing its structure. This cachable structure can later be resolved against actual interpolation values to produce a Node tree. """ + if config is None: + config = make_placeholder_config() parser = TemplateParser() - parser.feed_template(t) + parser.feed_template(t, placeholder_config=config) parser.close() return parser.get_tnode() From f85ab7c48bb9d654ae2af03c081dadc7625e3114 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Wed, 24 Jun 2026 14:06:26 -0700 Subject: [PATCH 22/78] Pull the placeholder config definition up to processor to keep parser positions consistent across parsings. --- tdom/parser.py | 14 ++++++++------ tdom/processor.py | 6 +++++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/tdom/parser.py b/tdom/parser.py index be623987..2498aab2 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -754,15 +754,17 @@ def feed_template( self.feed(content) @staticmethod - def parse(t: Template, config: PlaceholderConfig | None = None) -> TNode: + def parse(t: Template, placeholder_config: PlaceholderConfig | None = None) -> TNode: """ Parse a Template containing valid HTML and substitutions and return - a TNode tree representing its structure. This cachable structure can later - be resolved against actual interpolation values to produce a Node tree. + a cacheable TNode tree representing its structure. + + A placeholder config must be passed to keep parser positions consistent + between calls. """ - if config is None: - config = make_placeholder_config() + if placeholder_config is None: + placeholder_config = make_placeholder_config() parser = TemplateParser() - parser.feed_template(t, placeholder_config=config) + parser.feed_template(t, placeholder_config=placeholder_config) parser.close() return parser.get_tnode() diff --git a/tdom/processor.py b/tdom/processor.py index 318cf671..92eb3fd5 100644 --- a/tdom/processor.py +++ b/tdom/processor.py @@ -45,6 +45,7 @@ TTemplatedAttribute, TText, ) +from .placeholders import PlaceholderConfig, make_placeholder_config from .protocols import HasHTMLDunder from .scope import ScopedTemplate from .template_utils import TemplateRef @@ -527,8 +528,11 @@ def to_tnode(self, template: Template) -> TNode: ... @dataclass(frozen=True) class TemplateParserProxy(ITemplateParserProxy): + + placeholder_config: PlaceholderConfig = field(default_factory=make_placeholder_config) + def to_tnode(self, template: Template) -> TNode: - return TemplateParser.parse(template) + return TemplateParser.parse(template, placeholder_config=self.placeholder_config) @dataclass(frozen=True) From 52d9edd2b29ffd3fe973eb0afbdea5bb6ad280fd Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Wed, 24 Jun 2026 14:24:50 -0700 Subject: [PATCH 23/78] Add TTree to wrap root TNode coming from parser with metadata. --- tdom/parser.py | 20 ++++++++++++++++++-- tdom/processor.py | 45 +++++++++++++++++++++++++++++++-------------- tdom/tnodes.py | 12 +++++++++++- 3 files changed, 60 insertions(+), 17 deletions(-) diff --git a/tdom/parser.py b/tdom/parser.py index 2498aab2..7dffd2fe 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -28,6 +28,7 @@ TSpreadAttribute, TTemplatedAttribute, TText, + TTree, ) @@ -733,6 +734,13 @@ def get_tnode(self) -> TNode: # CONSIDER: or as an empty text node? return self.finalize_tag(self.root) + def get_ttree(self) -> TTree: + return TTree( + self.get_tnode(), + placeholder_config=self.get_source().placeholders.config, + sinfos=tuple(self.sinfo_table.values()), + ) + # ------------------------------------------ # Feeding and parsing # ------------------------------------------ @@ -754,7 +762,9 @@ def feed_template( self.feed(content) @staticmethod - def parse(t: Template, placeholder_config: PlaceholderConfig | None = None) -> TNode: + def parse( + t: Template, placeholder_config: PlaceholderConfig | None = None + ) -> TNode: """ Parse a Template containing valid HTML and substitutions and return a cacheable TNode tree representing its structure. @@ -762,9 +772,15 @@ def parse(t: Template, placeholder_config: PlaceholderConfig | None = None) -> T A placeholder config must be passed to keep parser positions consistent between calls. """ + return TemplateParser.parse_to_ttree(t, placeholder_config).root + + @staticmethod + def parse_to_ttree( + t: Template, placeholder_config: PlaceholderConfig | None = None + ) -> TTree: if placeholder_config is None: placeholder_config = make_placeholder_config() parser = TemplateParser() parser.feed_template(t, placeholder_config=placeholder_config) parser.close() - return parser.get_tnode() + return parser.get_ttree() diff --git a/tdom/processor.py b/tdom/processor.py index 92eb3fd5..2615599f 100644 --- a/tdom/processor.py +++ b/tdom/processor.py @@ -29,14 +29,18 @@ SVG_TAG_FIX, VOID_ELEMENTS, ) -from .parser import ( - HTMLAttribute, +from .parser import TemplateParser +from .parser_utils import HTMLAttribute +from .placeholders import PlaceholderConfig, make_placeholder_config +from .protocols import HasHTMLDunder +from .scope import ScopedTemplate +from .template_utils import TemplateRef +from .tnodes import ( TAttribute, TComment, TComponent, TDocumentType, TElement, - TemplateParser, TFragment, TInterpolatedAttribute, TLiteralAttribute, @@ -44,11 +48,8 @@ TSpreadAttribute, TTemplatedAttribute, TText, + TTree, ) -from .placeholders import PlaceholderConfig, make_placeholder_config -from .protocols import HasHTMLDunder -from .scope import ScopedTemplate -from .template_utils import TemplateRef from .utils import CachableTemplate, LastUpdatedOrderedDict type Attribute = tuple[str, object] @@ -524,26 +525,42 @@ def copy( class ITemplateParserProxy(t.Protocol): def to_tnode(self, template: Template) -> TNode: ... + def to_ttree(self, template: Template) -> TTree: ... @dataclass(frozen=True) class TemplateParserProxy(ITemplateParserProxy): + placeholder_config: PlaceholderConfig = field( + default_factory=make_placeholder_config + ) - placeholder_config: PlaceholderConfig = field(default_factory=make_placeholder_config) + def to_tnode(self, template: Template) -> TNode: # BWC + return TemplateParser.parse( + template, placeholder_config=self.placeholder_config + ) - def to_tnode(self, template: Template) -> TNode: - return TemplateParser.parse(template, placeholder_config=self.placeholder_config) + def to_ttree(self, template: Template) -> TTree: + return TemplateParser.parse_to_ttree( + template, placeholder_config=self.placeholder_config + ) @dataclass(frozen=True) class CachedTemplateParserProxy(TemplateParserProxy): @lru_cache(512) # noqa: B019 - def _to_tnode(self, ct: CachableTemplate) -> TNode: + def _to_tnode(self, ct: CachableTemplate) -> TNode: # BWC return super().to_tnode(ct.template) - def to_tnode(self, template: Template) -> TNode: + def to_tnode(self, template: Template) -> TNode: # BWC return self._to_tnode(CachableTemplate(template)) + @lru_cache(512) # noqa: B019 + def _to_ttree(self, ct: CachableTemplate) -> TTree: + return super().to_ttree(ct.template) + + def to_ttree(self, template: Template) -> TTree: + return self._to_ttree(CachableTemplate(template)) + class IComponentProcessor(t.Protocol): """Isolate component processing to allow for replacement.""" @@ -671,8 +688,8 @@ def process( return self._process_template(root_template, assume_ctx) def _process_template(self, template: Template, last_ctx: ProcessContext) -> str: - root = self.parser_api.to_tnode(template) - return self._process_tnode(template, last_ctx, root) + ttree = self.parser_api.to_ttree(template) + return self._process_tnode(template, last_ctx, ttree.root) def _process_tnode( self, template: Template, last_ctx: ProcessContext, tnode: TNode diff --git a/tdom/tnodes.py b/tdom/tnodes.py index cee6d6da..ef56a50e 100644 --- a/tdom/tnodes.py +++ b/tdom/tnodes.py @@ -1,7 +1,8 @@ import typing as t from dataclasses import dataclass, field -from .source import FrozenPosition +from .placeholders import PlaceholderConfig +from .source import FrozenPosition, TagSourceInfo from .template_utils import TemplateRef @@ -109,4 +110,13 @@ class TComponent(TNode): parser_pos: FrozenPosition | None = field(default=None, compare=False) +@dataclass +class TTree: + root: TNode + + placeholder_config: PlaceholderConfig + + sinfos: tuple[TagSourceInfo, ...] = () + + type TTag = TElement | TComponent | TFragment From b7ad50f9c93aa518f2c08f0f7fd4d72feed664d9 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Thu, 25 Jun 2026 08:15:44 -0700 Subject: [PATCH 24/78] Add helper method to unpack sinfos tuple into a mapping. --- tdom/tnodes.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tdom/tnodes.py b/tdom/tnodes.py index ef56a50e..935d86b6 100644 --- a/tdom/tnodes.py +++ b/tdom/tnodes.py @@ -118,5 +118,8 @@ class TTree: sinfos: tuple[TagSourceInfo, ...] = () + def unpack_sinfo_table(self) -> dict[FrozenPosition, TagSourceInfo]: + return {sinfo.starttag_pos: sinfo for sinfo in self.sinfos} + type TTag = TElement | TComponent | TFragment From c5fa872900892997bacbd89a29db7d9a4d8443cd Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Fri, 26 Jun 2026 12:22:47 -0700 Subject: [PATCH 25/78] Draft of adding custom exceptions. --- tdom/parser.py | 52 +++++---- tdom/processor.py | 274 ++++++++++++++++++++++++++++++++++++---------- 2 files changed, 247 insertions(+), 79 deletions(-) diff --git a/tdom/parser.py b/tdom/parser.py index 7dffd2fe..2f8ae932 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -32,6 +32,18 @@ ) +class ParsingError(Exception): + pass + + +class ParsingAssertionError(ParsingError): + pass + + +class AttributeParsingError(ParsingError): + pass + + @dataclass(frozen=True, slots=True) class OpenTagSourceInfo: """ @@ -220,11 +232,11 @@ def make_tattr(self, attr: HTMLAttribute) -> TAttribute: else: return TTemplatedAttribute(name=name, value_ref=value_ref) if value_ref is not None: - raise ValueError( + raise AttributeParsingError( "Attribute names cannot contain interpolations if the value is also interpolated." ) if not name_ref.is_singleton: - raise ValueError( + raise AttributeParsingError( "Spread attributes must have exactly one interpolation in the name." ) return TSpreadAttribute(i_index=name_ref.i_indexes[0]) @@ -259,7 +271,7 @@ def make_open_tag( return open_tag if not tag_ref.is_singleton: - raise ValueError( + raise ParsingError( "Component element tags must have exactly one interpolation." ) @@ -348,7 +360,7 @@ def compute_offset_into_children_start_s( temp_placeholders = PlaceholderState(known=known, config=config) tag_ref = temp_placeholders.remove_placeholders(starttag_text) if not temp_placeholders.is_empty: - raise AssertionError( + raise ParsingAssertionError( "There are extra placeholders still in the starttag_text." ) # Now the last string should terminate the starttag and end with ">" @@ -467,22 +479,22 @@ def validate_end_tag(self, tag: str, open_tag: OpenTag) -> int | None: match open_tag: case OpenTElement(): if not tag_ref.is_literal: - raise ValueError( + raise ParsingError( f"Component closing tag found for element <{open_tag.tag}>." ) if tag != open_tag.tag: - raise ValueError( + raise ParsingError( f"Mismatched closing tag for element <{open_tag.tag}>." ) return None case OpenTFragment(): - raise NotImplementedError("We do not support anonymous fragments.") + raise ParsingAssertionError("We do not support anonymous fragments.") case OpenTComponent(start_i_index=start_i_index): if tag_ref.is_literal: starttag = source.format_starttag(start_i_index) - e = ValueError( + e = ParsingError( f"Mismatched closing tag for component with tag {{{starttag}}}." ) if self.has_ambiguous_forward_slash(open_tag.sinfo): @@ -491,7 +503,7 @@ def validate_end_tag(self, tag: str, open_tag: OpenTag) -> int | None: ) raise e if not tag_ref.is_singleton: - raise ValueError( + raise ParsingError( "Component end tags must have exactly one interpolation." ) return tag_ref.i_indexes[0] @@ -504,7 +516,7 @@ def get_starttag_text(self, msg: str = "Expecting starttag text to be set.") -> """ starttag_text = super().get_starttag_text() if starttag_text is None: - raise AssertionError(msg) + raise ParsingAssertionError(msg) return starttag_text def has_ambiguous_forward_slash( @@ -559,15 +571,15 @@ def handle_endtag(self, tag: str) -> None: source = self.get_source() tag_ref = source.placeholders.copy().remove_placeholders(tag) if tag_ref.is_literal: - raise ValueError(f"Unexpected closing tag with no open tag.") + raise ParsingError(f"Unexpected closing tag with no open tag.") if not tag_ref.is_singleton: # @TODO: Also it doesn't match anything - raise ValueError( + raise ParsingError( "Component end tags must have exactly one interpolation." ) # Component tag endtag but no component tag is open... unmatched_endtag = self.get_source().format_endtag(tag_ref.i_indexes[0]) - raise ValueError( + raise ParsingError( f"Unexpected closing component tag with no open tag." ) open_tag = self.stack.pop() @@ -633,13 +645,13 @@ def handle_decl(self, decl: str) -> None: source = self.get_source() ref = source.placeholders.remove_placeholders(decl) if not ref.is_literal: - raise ValueError("Interpolations are not allowed in declarations.") + raise ParsingError("Interpolations are not allowed in declarations.") elif decl.upper().startswith("DOCTYPE "): doctype_content = decl[7:].strip() doctype = TDocumentType(doctype_content, parser_pos=self.get_parser_pos()) self.append_child(doctype) else: - raise NotImplementedError( + raise ParsingError( "Only well formed DOCTYPE declarations are currently supported." ) @@ -656,15 +668,15 @@ def close(self) -> None: if self.waiting_for_data(): # We apply heuristics here to try to guess why the parser didn't finish. if self.rawdata.count('"') % 2 == 1 or self.rawdata.count("'") % 2 == 1: - raise ValueError( + raise ParsingError( "Parser expects more data, maybe you left an attribute quote unclosed?" ) else: - raise ValueError( + raise ParsingError( "Parser expects more data, is the template valid html?" ) if self.stack: - e = ValueError("Invalid HTML structure: unclosed tags remain.") + e = ParsingError("Invalid HTML structure: unclosed tags remain.") # @TODO: We need to determine which tags this might apply to, # this only applies to components. parent = self.stack[-1] @@ -707,7 +719,7 @@ def close(self) -> None: ) raise e if not source.placeholders.is_empty: - raise ValueError("Some placeholders were never resolved.") + raise ParsingError("Some placeholders were never resolved.") super().close() def waiting_for_data(self): @@ -747,7 +759,7 @@ def get_ttree(self) -> TTree: def get_source(self) -> SourceTracker: if self.source is None: - raise AssertionError("Source has not been initialized.") + raise ParsingAssertionError("Source has not been initialized.") return self.source def feed_template( diff --git a/tdom/processor.py b/tdom/processor.py index 2615599f..03cddddf 100644 --- a/tdom/processor.py +++ b/tdom/processor.py @@ -29,7 +29,7 @@ SVG_TAG_FIX, VOID_ELEMENTS, ) -from .parser import TemplateParser +from .parser import ParsingError, TemplateParser from .parser_utils import HTMLAttribute from .placeholders import PlaceholderConfig, make_placeholder_config from .protocols import HasHTMLDunder @@ -61,6 +61,50 @@ # -------------------------------------------------------------------------- +@dataclass(frozen=True) +class TemplateErrorState: + template: Template + ttree: TTree | None = None + tnode: TNode | None = None + values_index: int | None = None + iter_index: int | None = None + + +class ProcessingError(Exception): + """General error when processing a template.""" + + last_tnode: TNode | None + " Nearest tnode from error if applicable. " + + template_e_states: list[TemplateErrorState] + " Stack of processor template error states if applicable. " + + values_index: int | None + " Index of the last failed interpolation. " + + iter_index: int | None + " Iteration of the last failed iterable value. " + + def __init__(self, msg: str = "") -> None: + super().__init__(msg) + self.template_e_states = [] + self.last_tnode = None + self.values_index = None + self.iter_index = None + + +class AttributeProcessingError(ProcessingError): + """Error while processing an element or component attribute.""" + + +class TextProcessingError(ProcessingError): + """Error while processing an element or component attribute.""" + + +class ComponentInvocationError(ProcessingError): + """Error while processing an element or component attribute.""" + + def _format_safe(value: object, format_spec: str) -> str: """Use Markup() to mark a value as safe HTML.""" assert format_spec == "safe" @@ -113,7 +157,7 @@ def _expand_aria_attr(value: object) -> Iterable[HTMLAttribute]: else: yield f"aria-{sub_k}", str(sub_v) else: - raise TypeError( + raise AttributeProcessingError( f"Cannot use {type(value).__name__} as value for aria attribute" ) @@ -129,7 +173,7 @@ def _expand_data_attr(value: object) -> Iterable[Attribute]: else: yield f"data-{sub_k}", str(sub_v) else: - raise TypeError( + raise AttributeProcessingError( f"Cannot use {type(value).__name__} as value for data attribute" ) @@ -147,7 +191,7 @@ def _substitute_spread_attrs(value: object) -> Iterable[Attribute]: elif isinstance(value, Mapping): yield from value.items() else: - raise TypeError( + raise AttributeProcessingError( f"Cannot use {type(value).__name__} as value for spread attributes" ) @@ -168,7 +212,7 @@ def parse_style_attribute_value(style_str: str) -> list[tuple[str, str | None]]: if prop: prop_parts = [p.strip() for p in prop.split(":") if p.strip()] if len(prop_parts) != 2: - raise ValueError( + raise AttributeProcessingError( f"Invalid number of parts for style property {prop} in {style_str}" ) styles.append((prop_parts[0], prop_parts[1])) @@ -187,7 +231,7 @@ def make_style_accumulator(old_value: object) -> StyleAccumulator: case True: # A bare attribute will just default to {}. styles = {} case _: - raise TypeError(f"Unexpected value: {old_value}") + raise AttributeProcessingError(f"Unexpected style value: {old_value}") return StyleAccumulator(styles=styles) @@ -214,7 +258,7 @@ def merge_value(self, value: object) -> None: case None: pass case _: - raise TypeError( + raise AttributeProcessingError( f"Unknown interpolated style value {value}, use '' to omit." ) @@ -240,7 +284,7 @@ def make_class_accumulator(old_value: object) -> ClassAccumulator: case True: toggled_classes = {} case _: - raise ValueError(f"Unexpected value {old_value}") + raise AttributeProcessingError(f"Unexpected class value {old_value}") return ClassAccumulator(toggled_classes=toggled_classes) @@ -269,11 +313,11 @@ def merge_value(self, value: object) -> None: pass case _: if item == value: - raise TypeError( + raise AttributeProcessingError( f"Unknown interpolated class value: {value}" ) else: - raise TypeError( + raise AttributeProcessingError( f"Unknown interpolated class item in {value}: {item}" ) @@ -347,7 +391,9 @@ def _resolve_t_attrs( ) new_attrs[name] = attr_accs[name].merge_value(attr_value) elif expander := ATTR_EXPANDERS.get(name): - raise TypeError(f"{name} attributes cannot be templated") + raise AttributeProcessingError( + f"{name} attributes cannot be templated" + ) else: new_attrs[name] = attr_value case TSpreadAttribute(i_index=i_index): @@ -366,7 +412,9 @@ def _resolve_t_attrs( else: new_attrs[sub_k] = sub_v case _: - raise ValueError(f"Unknown TAttribute type: {type(attr).__name__}") + raise AttributeProcessingError( + f"Unknown TAttribute type: {type(attr).__name__}" + ) for acc_name, acc in attr_accs.items(): # Skip "touching" the key here so that the order remains intact. super(type(new_attrs), new_attrs).__setitem__(acc_name, acc.to_value()) @@ -423,7 +471,7 @@ def _prep_component_kwargs( # We can't know what kwarg to put here... if raise_on_requires_positional and callable_info.requires_positional: - raise TypeError( + raise ComponentInvocationError( "Component callables cannot have required positional arguments." ) @@ -435,10 +483,12 @@ def _prep_component_kwargs( if snake_name in callable_info.named_params or callable_info.kwargs: kwargs[snake_name] = attr_value else: - raise ValueError(f"Unexpected attribute {snake_name}.") + raise ComponentInvocationError(f"Unexpected attribute {snake_name}.") if "children" in kwargs: - raise ValueError("The children attribute is reserved for component children.") + raise ComponentInvocationError( + "The children attribute is reserved for component children." + ) if "children" in callable_info.named_params: kwargs["children"] = children @@ -452,7 +502,7 @@ def _prep_component_kwargs( if raise_on_missing: missing = callable_info.required_named_params - kwargs.keys() if missing: - raise TypeError( + raise ComponentInvocationError( f"Missing required parameters for component: {', '.join(missing)}" ) @@ -625,30 +675,52 @@ def process( won't construct one directly. """ if not callable(component_callable): - raise TypeError( + raise ComponentInvocationError( f"Component callable must be callable: {type(component_callable)}" ) + try: + tattrs = _resolve_t_attrs(attrs, template.interpolations) + except ProcessingError: # @TODO: Is there a native way to guard this? + raise + except Exception as e: + # Causes: + # - Could be a failed "callable" formatter + # - Could be a failed "__html__()" call -- I think? # @TODO: + # + raise AttributeProcessingError( + "Error occurred processing component attributes" + ) from e kwargs = _prep_component_kwargs( get_callable_info(component_callable), - _resolve_t_attrs(attrs, template.interpolations), + tattrs, children=component_template, provided_attrs=provided_attrs, raise_on_requires_positional=True, raise_on_missing=True, ) - res1 = component_callable(**kwargs) # ty: ignore[call-top-callable] + try: + res1 = component_callable(**kwargs) # ty: ignore[call-top-callable] + except Exception as e: + raise ComponentInvocationError( + "Failed when invoking component callable." + ) from e if isinstance(res1, (Template, ScopedTemplate)): return res1 elif callable(res1): - res2 = res1() # ty: ignore[call-top-callable] + try: + res2 = res1() # ty: ignore[call-top-callable] + except Exception as e: + raise ComponentInvocationError( + "Failed when invoking component callable the second time." + ) from e if isinstance(res2, (Template, ScopedTemplate)): return res2 else: - raise TypeError( + raise ComponentInvocationError( f"Component object must return Template when called: {type(res2)}" ) else: - raise TypeError( + raise ComponentInvocationError( f"Component callable must return Template or Callable: {type(res1)}" ) @@ -685,11 +757,59 @@ def process( """ Process a TDOM compatible template into a string. """ - return self._process_template(root_template, assume_ctx) + try: + return self._process_template(root_template, assume_ctx) + except ProcessingError as e: + # + # @TODO: I think we could optionally consolidate and/or reformat + # all the exceptions here if needed and move this entire thing to + # a special error formatting tool. + # + for e_state in reversed(e.template_e_states): + if not e_state.ttree: + # Just skip this special case where processing could not + # even get started because the template wouldn't parse. + continue + sinfo_table = e_state.ttree.unpack_sinfo_table() + sinfo = parser_pos = None + if isinstance( + e_state.tnode, + (TElement, TText, TComment, TComponent, TDocumentType), + ): + parser_pos = e_state.tnode.parser_pos + if parser_pos: + sinfo = sinfo_table.get(parser_pos, None) + if sinfo: + e.add_note( + f"Error occurred at {type(e_state.tnode)} in template {sinfo.starttag_text} at {parser_pos}" + ) + else: + e.add_note(f"Error occurred at {type(e_state.tnode)} in template") + raise def _process_template(self, template: Template, last_ctx: ProcessContext) -> str: - ttree = self.parser_api.to_ttree(template) - return self._process_tnode(template, last_ctx, ttree.root) + try: + ttree = self.parser_api.to_ttree(template) + except ParsingError as parsing_e: + # Chain the parsing error into a processing error. + e = ProcessingError("Failed to parse template.") + e.template_e_states.append( + TemplateErrorState(template) + ) # Special case where nothing is set yet. + raise e from parsing_e + try: + return self._process_tnode(template, last_ctx, ttree.root) + except ProcessingError as e: + e.template_e_states.append( + TemplateErrorState( + template, ttree, e.last_tnode, e.values_index, e.iter_index + ) + ) + # Reset everything. + e.last_tnode = None + e.values_index = None + e.iter_index = None + raise def _process_tnode( self, template: Template, last_ctx: ProcessContext, tnode: TNode @@ -697,28 +817,35 @@ def _process_tnode( """ Process a tnode from a template's "t-tree" into a string. """ - match tnode: - case TDocumentType(text): - return self._process_document_type(last_ctx, text) - case TComment(ref): - return self._process_comment(template, last_ctx, ref) - case TFragment(children): - return self._process_fragment(template, last_ctx, children) - case TComponent(start_i_index, end_i_index, children_ref, attrs): - return self._process_component( - template, - last_ctx, - attrs, - start_i_index, - end_i_index, - children_ref, - ) - case TElement(tag, attrs, children): - return self._process_element(template, last_ctx, tag, attrs, children) - case TText(ref): - return self._process_texts(template, last_ctx, ref) - case _: - raise ValueError(f"Unrecognized tnode: {tnode}") + try: + match tnode: + case TDocumentType(text): + return self._process_document_type(last_ctx, text) + case TComment(ref): + return self._process_comment(template, last_ctx, ref) + case TFragment(children): + return self._process_fragment(template, last_ctx, children) + case TComponent(start_i_index, end_i_index, children_ref, attrs): + return self._process_component( + template, + last_ctx, + attrs, + start_i_index, + end_i_index, + children_ref, + ) + case TElement(tag, attrs, children): + return self._process_element( + template, last_ctx, tag, attrs, children + ) + case TText(ref): + return self._process_texts(template, last_ctx, ref) + case _: + raise ValueError(f"Unrecognized tnode: {tnode}") + except ProcessingError as e: + if e.last_tnode is None: + e.last_tnode = tnode + raise def _process_document_type( self, @@ -727,7 +854,7 @@ def _process_document_type( ) -> str: if last_ctx.ns != "html": # Nit - raise ValueError( + raise ProcessingError( "Cannot process document type in subtree of a foreign element." ) if self.uppercase_doctype: @@ -822,7 +949,14 @@ def _process_attrs( """ Process an element's attributes into a string. """ - resolved_attrs = _resolve_t_attrs(attrs, template.interpolations) + try: + resolved_attrs = _resolve_t_attrs(attrs, template.interpolations) + except ProcessingError: # @TODO: Is there a native way to guard this? + raise + except Exception as e: + raise AttributeProcessingError( + "Unexpected error occurred while processing element attrs." + ) from e if last_ctx.ns == "svg": attrs_str = serialize_html_attrs( _fix_svg_attrs(_resolve_html_attrs(resolved_attrs)) @@ -852,7 +986,7 @@ def _process_component( and template.interpolations[start_i_index].value != template.interpolations[end_i_index].value ): - raise TypeError( + raise ComponentInvocationError( "Component callable in start tag must match component callable in end tag." ) component_callable = template.interpolations[start_i_index].value @@ -888,7 +1022,7 @@ def _process_raw_texts( allow_markup=True, ) else: - raise NotImplementedError( + raise TextProcessingError( f"Parent tag {last_ctx.parent_tag} is not supported." ) @@ -935,13 +1069,17 @@ def _process_normal_text( """ value = format_interpolation(template.interpolations[values_index]) value = t.cast(NormalTextInterpolationValue, value) # ty: ignore[redundant-cast] - return self._process_normal_text_from_value(template, last_ctx, value) + return self._process_normal_text_from_value( + template, last_ctx, value, values_index=values_index + ) def _process_normal_text_from_value( self, template: Template, last_ctx: ProcessContext, value: NormalTextInterpolationValue, + values_index: int | None = None, + iter_index: int | None = None, ) -> str: """ Process a single value into a string as "normal text". @@ -956,18 +1094,36 @@ def _process_normal_text_from_value( # implementing HasHTMLDunder. return self.escape_html_text(value) elif isinstance(value, Template): - return self._process_template(value, last_ctx) + try: + return self._process_template(value, last_ctx) + except ProcessingError as e: + assert e.values_index is None and e.iter_index is None + e.values_index = values_index + e.iter_index = iter_index + raise elif isinstance(value, Iterable): return "".join( - self._process_normal_text_from_value(template, last_ctx, v) - for v in value + self._process_normal_text_from_value( + template, + last_ctx, + v, + iter_index=iter_index, + values_index=values_index, + ) + for iter_index, v in enumerate(value) ) elif isinstance(value, HasHTMLDunder): # @NOTE: markupsafe's escape does this for us but we put this in # here for completeness. # @NOTE: An actual Markup() would actually pass as a str() but a # custom object with __html__ might not. - return Markup(value.__html__()) + try: + return Markup(value.__html__()) + except Exception as e: + pe = TextProcessingError("Error occurred when processing text.") + pe.values_index = values_index + pe.iter_index = iter_index + raise pe from e else: # @DESIGN: Everything that isn't an object we recognize is # coerced to a str() and emitted. @@ -997,7 +1153,7 @@ def resolve_text_without_recursion( # the interpolation in this special case. return Markup(value.__html__()) elif isinstance(value, (Template, Iterable)): - raise ValueError( + raise TextProcessingError( f"Recursive includes are not supported within {parent_tag}" ) else: @@ -1019,11 +1175,11 @@ def resolve_text_without_recursion( if value: text.append(value) elif not isinstance(value, str) and isinstance(value, (Template, Iterable)): - raise ValueError( + raise TextProcessingError( f"Recursive includes are not supported within {parent_tag}" ) elif isinstance(value, HasHTMLDunder): - raise ValueError( + raise TextProcessingError( f"Non-exact trusted interpolations are not supported within {parent_tag}" ) else: From 4eabdd2416c3c3a5ccd7618a4264437e2e0a3b9f Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Fri, 26 Jun 2026 23:55:09 -0700 Subject: [PATCH 26/78] use regular assertion error for what would be our own bug --- tdom/parser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tdom/parser.py b/tdom/parser.py index 2f8ae932..3fd7dd94 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -759,7 +759,8 @@ def get_ttree(self) -> TTree: def get_source(self) -> SourceTracker: if self.source is None: - raise ParsingAssertionError("Source has not been initialized.") + # This would be a bug. + raise AssertionError("Source has not been initialized.") return self.source def feed_template( From 04ff538adcf7a8142cda45b090287d02e507ef01 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Fri, 26 Jun 2026 23:56:11 -0700 Subject: [PATCH 27/78] Update parser exception tests. --- tdom/parser_test.py | 64 +++++++++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/tdom/parser_test.py b/tdom/parser_test.py index 523b918f..e103cb16 100644 --- a/tdom/parser_test.py +++ b/tdom/parser_test.py @@ -2,7 +2,11 @@ import pytest -from .parser import TemplateParser +from .parser import ( + AttributeParsingError, + ParsingError, + TemplateParser, +) from .placeholders import make_placeholder_config from .source import FrozenPosition from .template_utils import TemplateRef @@ -209,17 +213,17 @@ def test_parse_title_unusual(): def test_parse_mismatched_tags(): - with pytest.raises(ValueError): + with pytest.raises(ParsingError, match="Mismatch"): _ = TemplateParser.parse(t"
Mismatched
") def test_parse_unclosed_tag(): - with pytest.raises(ValueError): + with pytest.raises(ParsingError, match="unclosed tags remain"): _ = TemplateParser.parse(t"
Unclosed") def test_parse_unexpected_closing_tag(): - with pytest.raises(ValueError): + with pytest.raises(ParsingError, match="Unexpected closing tag"): _ = TemplateParser.parse(t"Unopened
") @@ -243,12 +247,12 @@ def test_nested_self_closing_tags(): def test_self_closing_tags_unexpected_closing_tag(): - with pytest.raises(ValueError): + with pytest.raises(ParsingError, match="Unexpected closing tag"): _ = TemplateParser.parse(t"
") def test_self_closing_void_tags_unexpected_closing_tag(): - with pytest.raises(ValueError): + with pytest.raises(ParsingError, match="Unexpected closing tag"): _ = TemplateParser.parse(t"") @@ -339,20 +343,28 @@ def test_spread_attr(): def test_templated_attribute_name_error(): - with pytest.raises(ValueError): + with pytest.raises( + AttributeParsingError, + match="cannot contain interpolations if the value is also interpolated", + ): attr_name = "some-attr" _ = TemplateParser.parse(t'
') def test_templated_attribute_name_and_value_error(): - with pytest.raises(ValueError): + with pytest.raises( + AttributeParsingError, + match="cannot contain interpolations if the value is also interpolated", + ): attr_name = "some-attr" value = "value" _ = TemplateParser.parse(t'
') def test_adjacent_spread_attrs_error(): - with pytest.raises(ValueError): + with pytest.raises( + AttributeParsingError, match="must have exactly one interpolation in the name" + ): attrs1 = {} attrs2 = {} _ = TemplateParser.parse(t"
") @@ -384,14 +396,16 @@ def test_parse_doctype(): def test_parse_doctype_interpolation_error(): extra = "SYSTEM" - with pytest.raises(ValueError): + with pytest.raises( + ParsingError, match="Interpolations are not allowed in declarations" + ): _ = TemplateParser.parse(t"") def test_unsupported_decl_error(): - with pytest.raises(NotImplementedError): + with pytest.raises(ParsingError, match="Only well formed DOCTYPE declarations"): _ = TemplateParser.parse(t"") # Unknown declaration - with pytest.raises(NotImplementedError): + with pytest.raises(ParsingError, match="Only well formed DOCTYPE declarations"): _ = TemplateParser.parse(t"") # missing DTD @@ -441,7 +455,7 @@ def test_component_element_invalid_closing_tag(): def Component(): pass - with pytest.raises(ValueError): + with pytest.raises(ParsingError, match="Mismatched closing tag"): _ = TemplateParser.parse(t"<{Component}>
") @@ -449,7 +463,7 @@ def test_component_element_invalid_opening_tag(): def Component(): pass - with pytest.raises(ValueError): + with pytest.raises(ParsingError, match="Component closing tag found"): _ = TemplateParser.parse(t"
") @@ -457,7 +471,7 @@ def test_adjacent_start_component_tag_error(): def Component(): pass - with pytest.raises(ValueError): + with pytest.raises(ParsingError, match="must have exactly one interpolation"): _ = TemplateParser.parse(t"<{Component}{Component}>") @@ -465,7 +479,7 @@ def test_adjacent_end_component_tag_error(): def Component(): pass - with pytest.raises(ValueError): + with pytest.raises(ParsingError, match="must have exactly one interpolation"): _ = TemplateParser.parse(t"<{Component}>") @@ -473,7 +487,7 @@ def test_unmatched_end_component_tag_error(): def Component(): pass - with pytest.raises(ValueError, match="Unexpected closing component tag"): + with pytest.raises(ParsingError, match="Unexpected closing component tag"): _ = TemplateParser.parse(t"") @@ -496,17 +510,17 @@ def test_placeholder_collision_avoidance(): class TestIncompleteParsing: def test_dangling_quotes(self): - with pytest.raises(ValueError, match="Parser expects more data"): + with pytest.raises(ParsingError, match="Parser expects more data"): _ = TemplateParser.parse(t"
") def test_comp_unquoted_attr_value_error_nested_in_el(self, Comp): with pytest.raises( - ValueError, match="Did you mean to quote the last attribute" + ParsingError, match="Did you mean to quote the last attribute" ): _ = TemplateParser.parse(t"
<{Comp} title=today/>
") def test_comp_unquoted_attr_value_error_single_nested_in_comp(self, Comp, Comp2): with pytest.raises( - ValueError, match="Did you mean to quote the last attribute" + ParsingError, match="Did you mean to quote the last attribute" ): _ = TemplateParser.parse(t"<{Comp2}><{Comp} title=today/>") def test_comp_unquoted_attr_value_error_double_nested_in_comp(self, comp_maker): Comp1, Comp2, Comp3 = comp_maker("1"), comp_maker("2"), comp_maker("3") with pytest.raises( - ValueError, match="Did you mean to quote the last attribute" + ParsingError, match="Did you mean to quote the last attribute" ): _ = TemplateParser.parse( t"<{Comp2}><{Comp1}><{Comp3} title=today/>" From 0caca2e317225e575e10d58c16ac2abeab3b398e Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Sat, 27 Jun 2026 00:19:44 -0700 Subject: [PATCH 28/78] Update processor test exceptions. --- tdom/processor_test.py | 100 +++++++++++++++++++++++------------------ 1 file changed, 57 insertions(+), 43 deletions(-) diff --git a/tdom/processor_test.py b/tdom/processor_test.py index babd6a20..7f0bc759 100644 --- a/tdom/processor_test.py +++ b/tdom/processor_test.py @@ -13,10 +13,14 @@ from .callables import get_callable_info from .escaping import escape_html_text from .processor import ( + AttributeProcessingError, CachedTemplateParserProxy, + ComponentInvocationError, ProcessContext, + ProcessingError, TemplateParserProxy, TemplateProcessor, + TextProcessingError, _make_default_template_processor, ) from .processor import ( @@ -183,11 +187,11 @@ def test_templated_bool(self, bool_value): def test_templated_has_html_dunder_error(self, html_dunder_cls): """Objects with __html__ are not processed with literal text or other interpolations.""" text = html_dunder_cls("in a comment") - with pytest.raises(ValueError, match="not supported"): + with pytest.raises(TextProcessingError, match="not supported"): _ = html(t"") - with pytest.raises(ValueError, match="not supported"): + with pytest.raises(TextProcessingError, match="not supported"): _ = html(t"") - with pytest.raises(ValueError, match="not supported"): + with pytest.raises(TextProcessingError, match="not supported"): _ = html(t"") def test_templated_multiple_interpolations(self): @@ -207,12 +211,12 @@ def test_templated_escaping(self): def test_not_supported__recursive_template_error(self): text_t = t"comment" - with pytest.raises(ValueError, match="not supported"): + with pytest.raises(TextProcessingError, match="not supported"): _ = html(t"") def test_not_supported_recursive_iterable_error(self): texts = ["This", "is", "a", "comment"] - with pytest.raises(ValueError, match="not supported"): + with pytest.raises(TextProcessingError, match="not supported"): _ = html(t"") @@ -410,12 +414,12 @@ def test_style_with_content_escaped_in_normal_text(self): def test_not_supported_recursive_template_error(self): text_t = t"comment" - with pytest.raises(ValueError, match="not supported"): + with pytest.raises(TextProcessingError, match="not supported"): _ = html(t"") def test_not_supported_recursive_iterable_error(self): texts = ["This", "is", "a", "comment"] - with pytest.raises(ValueError, match="not supported"): + with pytest.raises(TextProcessingError, match="not supported"): _ = html(t"") @@ -520,7 +524,7 @@ def test_templated_object(self): ) def test_templated_has_html_dunder(self, html_dunder_cls): content = html_dunder_cls("anything") - with pytest.raises(ValueError, match="not supported"): + with pytest.raises(TextProcessingError, match="not supported"): _ = html(t"") def test_templated_escaping(self): @@ -538,12 +542,12 @@ def test_templated_multiple_interpolations(self): def test_not_supported_recursive_template_error(self): text_t = t"script" - with pytest.raises(ValueError, match="not supported"): + with pytest.raises(TextProcessingError, match="not supported"): _ = html(t"") def test_not_supported_recursive_iterable_error(self): texts = ["This", "is", "a", "script"] - with pytest.raises(ValueError, match="not supported"): + with pytest.raises(TextProcessingError, match="not supported"): _ = html(t"") @@ -625,7 +629,7 @@ def test_templated_object(self): ) def test_templated_has_html_dunder(self, html_dunder_cls): content = html_dunder_cls("anything") - with pytest.raises(ValueError, match="not supported"): + with pytest.raises(TextProcessingError, match="not supported"): _ = html(t"") def test_templated_escaping(self): @@ -647,22 +651,22 @@ def test_templated_multiple_interpolations(self): def test_exact_not_supported_recursive_template_error(self): text_t = t"style" - with pytest.raises(ValueError, match="not supported"): + with pytest.raises(TextProcessingError, match="not supported"): _ = html(t"") def test_inexact_not_supported_recursive_template_error(self): text_t = t"style" - with pytest.raises(ValueError, match="not supported"): + with pytest.raises(TextProcessingError, match="not supported"): _ = html(t"") def test_exact_not_supported_recursive_iterable_error(self): texts = ["This", "is", "a", "style"] - with pytest.raises(ValueError, match="not supported"): + with pytest.raises(TextProcessingError, match="not supported"): _ = html(t"") def test_inexact_not_supported_recursive_iterable_error(self): texts = ["This", "is", "a", "style"] - with pytest.raises(ValueError, match="not supported"): + with pytest.raises(TextProcessingError, match="not supported"): _ = html(t"") @@ -736,7 +740,7 @@ def test_templated_object(self): ) def test_templated_has_html_dunder(self, html_dunder_cls): content = html_dunder_cls("No") - with pytest.raises(ValueError, match="not supported"): + with pytest.raises(TextProcessingError, match="not supported"): _ = html(t"Literal html?: {content}") def test_templated_escaping(self): @@ -754,22 +758,22 @@ def test_templated_multiple_interpolations(self): def test_exact_not_supported_recursive_template_error(self): text_t = t"title" - with pytest.raises(ValueError, match="not supported"): + with pytest.raises(TextProcessingError, match="not supported"): _ = html(t"{text_t}") def test_exact_not_supported_recursive_iterable_error(self): texts = ["This", "is", "a", "title"] - with pytest.raises(ValueError, match="not supported"): + with pytest.raises(TextProcessingError, match="not supported"): _ = html(t"{texts}") def test_inexact_not_supported_recursive_template_error(self): text_t = t"title" - with pytest.raises(ValueError, match="not supported"): + with pytest.raises(TextProcessingError, match="not supported"): _ = html(t"{text_t} and more") def test_inexact_not_supported_recursive_iterable_error(self): texts = ["This", "is", "a", "title"] - with pytest.raises(ValueError, match="not supported"): + with pytest.raises(TextProcessingError, match="not supported"): _ = html(t"{texts} and more") @@ -850,7 +854,7 @@ def test_templated_object(self): ) def test_templated_has_html_dunder(self, html_dunder_cls): content = html_dunder_cls("No") - with pytest.raises(ValueError, match="not supported"): + with pytest.raises(TextProcessingError, match="not supported"): _ = html(t"") def test_templated_multiple_interpolations(self): @@ -868,12 +872,12 @@ def test_templated_escaping(self): def test_not_supported_recursive_template_error(self): text_t = t"textarea" - with pytest.raises(ValueError, match="not supported"): + with pytest.raises(TextProcessingError, match="not supported"): _ = html(t"") def test_not_supported_recursive_iterable_error(self): texts = ["This", "is", "a", "textarea"] - with pytest.raises(ValueError, match="not supported"): + with pytest.raises(TextProcessingError, match="not supported"): _ = html(t"") @@ -1001,13 +1005,17 @@ def get_value(): == f"<{tag}>The value is dynamic." ) + @pytest.mark.skip def test_callback_nonzero_callable_error(self): def add(a, b): return a + b assert add(1, 2) == 3, "Make sure fixture could work..." - with pytest.raises(TypeError): + with pytest.raises( + ProcessingError, + match="Should we wrap every call to format_interpolation and chain the exception?", + ): for tag in ("p", "script", "style"): _ = html( Template(f"<{tag}>") @@ -1117,7 +1125,7 @@ def test_spread_attr_none(self): def test_spread_attr_type_errors(self): for attrs in (0, [], (), False, True): - with pytest.raises(TypeError): + with pytest.raises(AttributeProcessingError): _ = html(t"") @@ -1228,7 +1236,7 @@ def test_data_attr_unrelated_unaffected(self): def test_data_attr_templated_error(self): data1 = {"user-id": "user-123"} data2 = {"role": "admin"} - with pytest.raises(TypeError): + with pytest.raises(AttributeProcessingError): _ = html(t'
') def test_data_attr_none(self): @@ -1238,7 +1246,7 @@ def test_data_attr_none(self): def test_data_attr_errors(self): for v in [False, [], (), 0, "data?"]: - with pytest.raises(TypeError): + with pytest.raises(AttributeProcessingError): _ = html(t"") def test_data_literal_attr_bypass(self): @@ -1255,7 +1263,7 @@ class TestSpecialAriaAttribute: def test_aria_templated_attr_error(self): aria1 = {"label": "close"} aria2 = {"hidden": "true"} - with pytest.raises(TypeError): + with pytest.raises(AttributeProcessingError): _ = html(t'
') def test_interpolated_mapping(self): @@ -1277,7 +1285,7 @@ def test_aria_interpolate_attr_none(self): def test_aria_attr_errors(self): for v in [False, [], (), 0, "aria?"]: - with pytest.raises(TypeError): + with pytest.raises(AttributeProcessingError): _ = html(t"") def test_aria_literal_attr_bypass(self): @@ -1359,9 +1367,9 @@ def test_class_none_ignored(self): def test_class_type_errors(self): for class_item in (False, True, 0): - with pytest.raises(TypeError): + with pytest.raises(AttributeProcessingError): _ = html(t"

") - with pytest.raises(TypeError): + with pytest.raises(AttributeProcessingError): _ = html(t"

") def test_class_merge_literals(self): @@ -1432,7 +1440,7 @@ def test_interpolated_style_attribute_multiple_placeholders(self): # CONSIDER: Is this what we want? Currently, when we have multiple # placeholders in a single attribute, we treat it as a string attribute # which produces an invalid style attribute. - with pytest.raises(ValueError): + with pytest.raises(AttributeProcessingError): _ = html(t"

Warning!

") def test_interpolated_style_attribute_merged(self): @@ -1453,7 +1461,7 @@ def test_style_attribute_str(self): assert res == '

Warning!

' def test_style_attribute_non_str_non_dict(self): - with pytest.raises(TypeError): + with pytest.raises(AttributeProcessingError): styles = [1, 2] _ = html(t"

Warning!

") @@ -1511,7 +1519,7 @@ def InputElement(size=10, type="text"): pass callable_info = get_callable_info(InputElement) - with pytest.raises(ValueError): + with pytest.raises(ComponentInvocationError): assert ( prep_component_kwargs(callable_info, {"type2": 15}, children=t"") == {} ) @@ -1555,7 +1563,9 @@ def Comp(children: Template) -> Template: return t"
{children}
" callable_info = get_callable_info(Comp) - with pytest.raises(ValueError, match="The children attribute is reserved"): + with pytest.raises( + ComponentInvocationError, match="The children attribute is reserved" + ): _ = prep_component_kwargs( callable_info, {"children": t""}, children=t"" ) @@ -1597,7 +1607,7 @@ def test_with_no_children(self): ) def test_missing_props_error(self): - with pytest.raises(TypeError): + with pytest.raises(ComponentInvocationError): _ = html( t"<{self.FunctionComponent}>Missing props" ) @@ -1823,14 +1833,14 @@ def AttributeTypeComponent( class TestComponentErrors: def test_component_non_callable_fails(self): - with pytest.raises(TypeError): + with pytest.raises(ComponentInvocationError): _ = html(t"<{'not a function'} />") def test_component_requiring_positional_arg_fails(self): def RequiresPositional(whoops: int, /) -> Template: # pragma: no cover return t"

Positional arg: {whoops}

" - with pytest.raises(TypeError): + with pytest.raises(ComponentInvocationError): _ = html(t"<{RequiresPositional} />") def test_mismatched_component_closing_tag_fails(self): @@ -1840,7 +1850,7 @@ def OpenTag(children: Template) -> Template: def CloseTag(children: Template) -> Template: return t"
close
" - with pytest.raises(TypeError): + with pytest.raises(ComponentInvocationError): _ = html(t"<{OpenTag}>Hello") @pytest.mark.parametrize( @@ -1851,7 +1861,8 @@ def BadFunctionComp(children: Template): return bad_value with pytest.raises( - TypeError, match="Component callable must return Template or Callable:" + ComponentInvocationError, + match="Component callable must return Template or Callable:", ): _ = html(t"<{BadFunctionComp}>Hello") @@ -1866,7 +1877,8 @@ def component_object(): return component_object with pytest.raises( - TypeError, match="Component object must return Template when called:" + ComponentInvocationError, + match="Component object must return Template when called:", ): _ = html(t"<{BadFactoryComp}>Hello") @@ -2099,7 +2111,8 @@ def test_dynamic_raw_text(self): content = '' content_t = t"{content}" with pytest.raises( - ValueError, match="Recursive includes are not supported within script" + TextProcessingError, + match="Recursive includes are not supported within script", ): content_t = t'' _ = html(t"") @@ -2109,7 +2122,8 @@ def test_dynamic_escapable_raw_text(self): content = '' content_t = t"{content}" with pytest.raises( - ValueError, match="Recursive includes are not supported within textarea" + TextProcessingError, + match="Recursive includes are not supported within textarea", ): _ = html(t"") From 809fda7ff7df131f0fa2fd8f19f8d78e75ec013a Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Sun, 28 Jun 2026 10:27:25 -0700 Subject: [PATCH 29/78] Add draft source reader for error reporting. --- tdom/parser.py | 10 +- tdom/source.py | 261 +++++++++++++++++++++++++++++++++++++++++ tdom/template_utils.py | 56 +++++++++ 3 files changed, 326 insertions(+), 1 deletion(-) diff --git a/tdom/parser.py b/tdom/parser.py index 3fd7dd94..8c661881 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -12,6 +12,7 @@ ) from .source import ( FrozenPosition, + SourceReader, TagSourceInfo, ) from .template_utils import TemplateRef, combine_template_refs @@ -167,6 +168,11 @@ def format_starttag(self, i_index: int) -> str: def format_endtag(self, i_index: int) -> str: return self.get_expression(i_index, fallback_prefix="component-endtag") + def get_reader(self) -> SourceReader: + return SourceReader( + template=self.template, placeholder_config=self.placeholders.config + ) + class TemplateParser(HTMLParser): root: OpenTFragment @@ -571,7 +577,9 @@ def handle_endtag(self, tag: str) -> None: source = self.get_source() tag_ref = source.placeholders.copy().remove_placeholders(tag) if tag_ref.is_literal: - raise ParsingError(f"Unexpected closing tag with no open tag.") + reader = source.get_reader() + pos_msg = reader.make_template_pos_msg(self.get_parser_pos()) + raise ParsingError(f"Unexpected closing tag with no open tag, {pos_msg}.") if not tag_ref.is_singleton: # @TODO: Also it doesn't match anything raise ParsingError( diff --git a/tdom/source.py b/tdom/source.py index 7ff90726..ecafca9a 100644 --- a/tdom/source.py +++ b/tdom/source.py @@ -1,6 +1,10 @@ +import typing as t from dataclasses import dataclass +from string.templatelib import Interpolation, Template from .parser_utils import HTMLAttribute +from .placeholders import PlaceholderConfig +from .template_utils import TemplateRef @dataclass(slots=True, frozen=True) @@ -45,3 +49,260 @@ class TagSourceInfo: " Position of the parser when the element starttag was parsed. " endtag_pos: FrozenPosition | None = None " Position of the parser when the element endtag was parsed. " + + +@dataclass(frozen=True) +class MultiPosition: + """ + @NOTE: Like other position tools this does not support positioning within + an interpolation. Either you are at the start of interpolation or at an + offset within a string. + """ + + pos: FrozenPosition + tpos: FrozenPosition + s_index: int + s_offset: int + i_index: int + + +def iterate_template_from_s_index(template: Template, s_index: int): + last_index = len(template.strings) - 1 + if s_index < 0 or s_index > last_index: + return + index = 0 + while index <= last_index: + yield template.strings[index] + if index < last_index: + yield template.interpolations[index] + index += 1 + + +def iterate_template_from_i_index(template: Template, i_index: int): + last_index = len(template.strings) - 1 + if i_index < 0 or i_index >= last_index: + return + index = 0 + while index <= last_index: + if index < last_index: + yield template.interpolations[index] + index += 1 + yield template.strings[index] + + +def template_repr_iter(template: Template) -> t.Generator[str]: + for part in template: + if isinstance(part, str): + yield part + else: + yield interpolation_repr(part) + + +def template_repr(template: Template) -> str: + return "".join(template_repr_iter(template)) + + +def interpolation_repr(ip: Interpolation) -> str: + expr_str = ip.expression + conversion_str = f"!{ip.conversion}" if ip.conversion is not None else "" + format_spec_str = f":{ip.format_spec}" if ip.format_spec else "" + return f"{{{expr_str}{conversion_str}{format_spec_str}}}" + + +@dataclass +class SourceReader: + "Format report-like strings from template source for the parser." + + template: Template + + placeholder_config: PlaceholderConfig + + def ref_to_repr(self, ref: TemplateRef, limit: int | None = None) -> str: + filled_template = ref.resolve(self.template.interpolations) + return template_repr(filled_template)[:limit] + + def make_template_pos_msg(self, parser_pos: FrozenPosition) -> str: + template_pos = self.to_template_pos(parser_pos) + return f"line {template_pos.line} offset {template_pos.offset}" + + def to_template_pos(self, parser_pos: FrozenPosition) -> FrozenPosition: + mpos = self._compute_positions(parser_pos) + return mpos.tpos + + def make_template_slice(self, parser_pos: FrozenPosition, limit: int | None = None): + mpos = self._compute_positions(parser_pos) + if mpos.s_index > mpos.i_index: + sub_template = Template( + *iterate_template_from_s_index(self.template, mpos.s_index) + ) + return template_repr(sub_template)[mpos.s_offset : limit] + else: + sub_template = Template( + *iterate_template_from_i_index(self.template, mpos.i_index) + ) + return template_repr(sub_template)[:limit] + + def _compute_positions(self, parser_pos: FrozenPosition) -> MultiPosition: + return compute_template_positions( + self.template, self.placeholder_config, parser_pos + ) + + +def compute_template_positions( + template: Template, + placeholder_config: PlaceholderConfig, + parser_pos: FrozenPosition, +) -> MultiPosition: + """ + Translate the given parser (pos)ition into template (pos)ition. + + @NOTE: There can be newlines in an interpolation expression which + results in the parser position's line being less than the + template position's line since a placeholder will not contain newlines. + + @NOTE: Similarly an interpolation as displayed can be longer than a + placeholder OR shorter than a placeholder causing the offsets to go + out of sync. + + @NOTE: There is a weird issue with `format_spec` and the specification + where you can't tell if a ':' was used or not when the `format_spec` is + empty. We just assume no one would leave it in without a non-empty + format_spec, ie. t"{val:}" would not exist even though it is valid. The + conversion does not have this issue because "{val!}" is invalid and when + no conversion is set the conversion value is None. + """ + # + # Walk until we reach the given parser pos, keeping both parser position + # and template position in sync. When the given parser position is reached + # then return the synced up template position. + # + pos = Position() + tpos = Position() + last_s_index = len(template.strings) - 1 + for s_index in range(len(template.strings)): + # + # Walk through `strings[s_index]` + # + s = template.strings[s_index] + if parser_pos.line > pos.line: + # need more lines + nls_found = s.count("\n") # how many were found? + nls_need = parser_pos.line - pos.line # how many are needed? + if nls_found >= nls_need: + pos.line += nls_need + tpos.line += nls_need + offset_found = len(s.split("\n", nls_need + 1)[nls_need]) + if offset_found >= parser_pos.offset: + # needed lines, found lines, found offset + tpos.offset = pos.offset = parser_pos.offset + total_offset = ( + sum( + len(line) + 1 + for line in s.split("\n", nls_need + 1)[:nls_need] + ) + + parser_pos.offset + ) + return MultiPosition( + pos=parser_pos, + tpos=tpos.freeze(), + s_index=s_index, + i_index=s_index - 1, + s_offset=total_offset, + ) + else: + # got enough lines, still need more offset + tpos.offset = pos.offset = offset_found + elif nls_found > 0: + # some lines but still need more lines + pos.line += nls_found + tpos.line += nls_found + tpos.offset = pos.offset = len(s[s.rfind("\n") + 1 :]) + else: + # no lines, still need more lines + offset_found = len(s) + tpos.offset += offset_found + pos.offset += offset_found + elif parser_pos.line == pos.line: + # got enough lines, we just need more offset + offset_found = len(s[: s.find("\n")]) if "\n" in s else len(s) + offset_need = parser_pos.offset - pos.offset + if offset_found >= offset_need: + pos.offset += offset_need + tpos.offset += offset_need + total_offset = offset_need # only from the start of this string. + # had lines, found offset + return MultiPosition( + pos=parser_pos, + tpos=tpos.freeze(), + s_index=s_index, + i_index=s_index - 1, + s_offset=total_offset, + ) + else: + tpos.offset += offset_found + pos.offset += offset_found + else: + # We should have dropped out and failed earlier this would be a bug. + raise AssertionError( + f"Unexpected line: {pos.line} greater than asked for {parser_pos.line}" + ) + + # + # Walk through `interpolations[s_index]` + # + if s_index < last_s_index: + ph_length = len(placeholder_config.make_placeholder(s_index)) + if ( + pos.line == parser_pos.line + and pos.offset + ph_length > parser_pos.offset + ): + # Ie. we don't know how to determine how much of the + # interpolation expression would be equivalent to + # a substring of a placeholder. + raise ValueError( + f"Cannot split a placeholder for interpolations[{s_index}], placeholders are atomic." + ) + + ip = template.interpolations[s_index] + expr = ip.expression + expr_line_count = expr.count("\n") + tpos.line += expr_line_count + pos.offset += ph_length + EXCLAIMATION_POINT = CONVERSION_CHAR = SEMICOLON = LEFT_CURLY_BRACE = ( + RIGHT_CURLY_BRACE + ) = 1 + tail = ( + ( + EXCLAIMATION_POINT + CONVERSION_CHAR + if ip.conversion is not None + else 0 + ) # "!" and conversion char or neither + + (SEMICOLON if ip.format_spec else 0) # ":" or not + + len(ip.format_spec) + + RIGHT_CURLY_BRACE + ) + if expr_line_count > 0: + tpos.offset = len(expr[expr.rfind("\n") + 1 :]) + tail + else: + tpos.offset += LEFT_CURLY_BRACE + len(expr) + tail + if pos == parser_pos: + return MultiPosition( + pos=parser_pos, + tpos=tpos.freeze(), + s_index=s_index, + i_index=s_index, + s_offset=0, + ) + if pos == parser_pos: + # @TODO: When can this fall through happen? Or is this always an error? + return MultiPosition( + pos=parser_pos, + tpos=tpos.freeze(), + s_index=len(template.strings) - 1, + i_index=len(template.strings) - 2, + s_offset=len(template.strings[-1]), + ) + else: + raise ValueError( + "Unexpected position {pos}, did not reach required position {parser_pos}" + ) diff --git a/tdom/template_utils.py b/tdom/template_utils.py index f5e3f060..096d3351 100644 --- a/tdom/template_utils.py +++ b/tdom/template_utils.py @@ -91,3 +91,59 @@ def resolve(self, interpolations: tuple[Interpolation, ...]) -> Template: """Use the given interpolations to resolve this reference template into a Template.""" resolved = [interpolations[i_index] for i_index in self.i_indexes] return template_from_parts(self.strings, resolved) + + +def slice_from_template( + template: Template, tslice: TemplateSlice +) -> t.Generator[Interpolation | str]: + """ + Yield the template parts that make up the requested slice. + """ + if tslice.start is None: + first = 0 + else: + first = tslice.start + assert first >= 0 and first < len(template.strings) + if tslice.start_offset is None: + offset = None + else: + offset = tslice.start_offset + if tslice.stop is None: + last = len(template.strings) - 1 + else: + last = tslice.stop - 1 + assert last >= 0 and last < len(template.strings) + if tslice.stop_limit is None: + limit = None + else: + limit = tslice.stop_limit + + if first == last: + yield template.strings[first][offset:limit] + return + else: + yield template.strings[first][offset:] + yield template.interpolations[first] + + for index in range(first + 1, last + 1): + if index == last: + yield template.strings[last][:limit] + else: + yield template.strings[index] + yield template.interpolations[index] + + +@dataclass(frozen=True, slots=True) +class TemplateSlice: + """ + strings[start][start_offset:] + ... + strings[stop][:stop_limit] + + @NOTE: Start offset could be len(string[start]) and likewise stop_limit could be 0. + """ + + start: int | None = None + start_offset: int | None = None + stop: int | None = None + stop_limit: int | None = None From 48756cc1d6bd9668931e245422b4500e0f94d46c Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Sun, 28 Jun 2026 10:27:47 -0700 Subject: [PATCH 30/78] Add some parser error reporting examples. --- tdom/parser.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/tdom/parser.py b/tdom/parser.py index 8c661881..cec91541 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -485,8 +485,20 @@ def validate_end_tag(self, tag: str, open_tag: OpenTag) -> int | None: match open_tag: case OpenTElement(): if not tag_ref.is_literal: + # + # EXAMPLE 3: getting earlier source slice and line number AND + # current source slice and current line number + # + reader = source.get_reader() + starttag_ref = reader.placeholder_config.find_placeholders( + open_tag.sinfo.starttag_text + ) + starttag_repr = reader.ref_to_repr(starttag_ref) + endtag_repr = reader.ref_to_repr(tag_ref) + endtag_pos_msg = reader.make_template_pos_msg(self.get_parser_pos()) + starttag_pos_msg = reader.make_template_pos_msg(open_tag.parser_pos) raise ParsingError( - f"Component closing tag found for element <{open_tag.tag}>." + f"Component closing tag at {endtag_pos_msg} found for element {starttag_repr} at {starttag_pos_msg}." ) if tag != open_tag.tag: raise ParsingError( @@ -577,18 +589,24 @@ def handle_endtag(self, tag: str) -> None: source = self.get_source() tag_ref = source.placeholders.copy().remove_placeholders(tag) if tag_ref.is_literal: + # EXAMPLE 1: getting line number, does not slice/extract source reader = source.get_reader() pos_msg = reader.make_template_pos_msg(self.get_parser_pos()) - raise ParsingError(f"Unexpected closing tag with no open tag, {pos_msg}.") + raise ParsingError( + f"Unexpected closing tag with no open tag, {pos_msg}." + ) if not tag_ref.is_singleton: # @TODO: Also it doesn't match anything raise ParsingError( "Component end tags must have exactly one interpolation." ) # Component tag endtag but no component tag is open... - unmatched_endtag = self.get_source().format_endtag(tag_ref.i_indexes[0]) + # EXAMPLE 2: getting line number AND get source (repr)esentation via ref + reader = source.get_reader() + pos_msg = reader.make_template_pos_msg(self.get_parser_pos()) + tag_repr = reader.ref_to_repr(tag_ref) raise ParsingError( - f"Unexpected closing component tag with no open tag." + f"Unexpected closing component tag with no open tag, {pos_msg}." ) open_tag = self.stack.pop() endtag_i_index = self.validate_end_tag(tag, open_tag) From 2e2bed4cd152dc915461aec85d333176a43dd72c Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Sun, 28 Jun 2026 10:39:35 -0700 Subject: [PATCH 31/78] Add processor error reporting example. --- tdom/processor.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tdom/processor.py b/tdom/processor.py index 03cddddf..e21806a1 100644 --- a/tdom/processor.py +++ b/tdom/processor.py @@ -34,6 +34,7 @@ from .placeholders import PlaceholderConfig, make_placeholder_config from .protocols import HasHTMLDunder from .scope import ScopedTemplate +from .source import SourceReader from .template_utils import TemplateRef from .tnodes import ( TAttribute, @@ -780,10 +781,22 @@ def process( if parser_pos: sinfo = sinfo_table.get(parser_pos, None) if sinfo: + # + # Example 4: Getting starttag repr and pos in processor. + # + reader = SourceReader( + e_state.template, + placeholder_config=e_state.ttree.placeholder_config, + ) + starttag_repr = reader.ref_to_repr( + reader.placeholder_config.find_placeholders(sinfo.starttag_text) + ) + starttag_pos_msg = reader.make_template_pos_msg(sinfo.starttag_pos) e.add_note( - f"Error occurred at {type(e_state.tnode)} in template {sinfo.starttag_text} at {parser_pos}" + f"Error occurred at {starttag_repr} at {starttag_pos_msg}." ) else: + # @TODO: Scrape together what we can for a better message. e.add_note(f"Error occurred at {type(e_state.tnode)} in template") raise From 93fd99a9fd2528f2979c946c6bd0602adf39479a Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Sun, 28 Jun 2026 12:56:07 -0700 Subject: [PATCH 32/78] Fix test for now. --- tdom/parser_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tdom/parser_test.py b/tdom/parser_test.py index e103cb16..a93e9c01 100644 --- a/tdom/parser_test.py +++ b/tdom/parser_test.py @@ -463,7 +463,9 @@ def test_component_element_invalid_opening_tag(): def Component(): pass - with pytest.raises(ParsingError, match="Component closing tag found"): + with pytest.raises( + ParsingError, match="Component closing tag .* found for element" + ): _ = TemplateParser.parse(t"
") From 26be511afaa787da0f156c11443d93bc96d234ff Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Tue, 30 Jun 2026 22:13:49 -0700 Subject: [PATCH 33/78] Use the starttag ref directly to compute start of children body. --- tdom/parser.py | 65 +++++++------------------------------------------- 1 file changed, 9 insertions(+), 56 deletions(-) diff --git a/tdom/parser.py b/tdom/parser.py index cec91541..d9ffdf33 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -302,22 +302,21 @@ def make_open_tag( starttag_text = self.get_starttag_text( f"Expected startag_text to be set when parsing component at {i_index}." ) - - tattrs = self.make_tattrs(attrs) - - offset_into_children_start_s = self.compute_offset_into_children_start_s( - start_i_index=i_index, - tattrs=tattrs, - config=source.placeholders.config, - starttag_text=starttag_text, - ) + # Bypass placeholder tracking and just make ref directly. + # Placeholder tracking should be covered by the SourceTracker independently. + starttag_ref = source.placeholders.config.find_placeholders(starttag_text) + # @NOTE: The last string should terminate the starttag and end with ">" + # So this length is the offset from the last interpolation to the start + # of the children's leading string. + offset_into_children_start_s = len(starttag_ref.strings[-1]) parser_pos = self.get_parser_pos() + open_tag = OpenTComponent( start_i_index=i_index, children_start_s_index=children_start_s_index, offset_into_children_start_s=offset_into_children_start_s, - attrs=tattrs, + attrs=self.make_tattrs(attrs), parser_pos=parser_pos, sinfo=OpenTagSourceInfo( starttag_text=starttag_text, @@ -328,52 +327,6 @@ def make_open_tag( ) return open_tag - def compute_offset_into_children_start_s( - self, - start_i_index: int, - tattrs: tuple[TAttribute, ...], - config: PlaceholderConfig, - starttag_text: str, - ) -> int: - """ - Compute offset into "string" containing the start of children template. - - @NOTE: This is to actually OFFLOAD work to the parser itself. If we try - to "rebuild" the tag from the parse result we are bound to fail in some - way(s). We essentially re-run the placeholder process but with content - we KNOWN ends at the end of the starttag, ie. ">", because the parser - told us that is where it ends (rather than trying to scan for ">" - because ">" might be in literal tags). - - Examples: - - <{Comp}> -- len(">") - <{Comp}>children -- len(">") - <{Comp} title="1>0">children -- len(' title="1>0">') - <{Comp} title="{'1>0'}">children -- len('">') - """ - # Rebuild known interpolations in the starttag. - known: set[int] = {start_i_index} # The component callable itself. - for attr in tattrs: - if isinstance(attr, TInterpolatedAttribute): - known.add(attr.value_i_index) - elif isinstance(attr, TSpreadAttribute): - known.add(attr.i_index) - elif isinstance(attr, TTemplatedAttribute): - known.update(attr.value_ref.i_indexes) - # Now re-remove those placeholders using the same config we used to - # make them. - temp_placeholders = PlaceholderState(known=known, config=config) - tag_ref = temp_placeholders.remove_placeholders(starttag_text) - if not temp_placeholders.is_empty: - raise ParsingAssertionError( - "There are extra placeholders still in the starttag_text." - ) - # Now the last string should terminate the starttag and end with ">" - # So this length is the offset from the last interpolation to the start - # of the children's leading string. - return len(tag_ref.strings[-1]) - def finalize_tag( self, open_tag: OpenTag, From 18b90e4cea8b9758117dbda23690d228712a83b0 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Tue, 30 Jun 2026 22:44:07 -0700 Subject: [PATCH 34/78] Use proxy methods on SourceTracker. --- tdom/parser.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/tdom/parser.py b/tdom/parser.py index d9ffdf33..01fabe5d 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -173,6 +173,23 @@ def get_reader(self) -> SourceReader: template=self.template, placeholder_config=self.placeholders.config ) + def remove_placeholders(self, text: str) -> TemplateRef: + """ + Find tracked placeholders in text and mark them as found. + + @NOTE: Raises if any untracked placeholders are found. + + If you want to make a TemplateRef without changing state use + `self.find_placeholders()`. + """ + return self.placeholders.remove_placeholders(text) + + def find_placeholders(self, text: str) -> TemplateRef: + """ + Find all placeholders without affecting tracking. + """ + return self.placeholders.config.find_placeholders(text) + class TemplateParser(HTMLParser): root: OpenTFragment @@ -540,7 +557,7 @@ def handle_startendtag(self, tag: str, attrs: Sequence[HTMLAttribute]) -> None: def handle_endtag(self, tag: str) -> None: if not self.stack: source = self.get_source() - tag_ref = source.placeholders.copy().remove_placeholders(tag) + tag_ref = source.find_placeholders(tag) if tag_ref.is_literal: # EXAMPLE 1: getting line number, does not slice/extract source reader = source.get_reader() @@ -602,7 +619,7 @@ def get_closed_tcomps( def handle_data(self, data: str) -> None: source = self.get_source() - ref = source.placeholders.remove_placeholders(data) + ref = source.remove_placeholders(data) parent = self.get_parent() if parent.children and isinstance(parent.children[-1], TText): prior_text = parent.children[-1] @@ -616,13 +633,13 @@ def handle_data(self, data: str) -> None: def handle_comment(self, data: str) -> None: source = self.get_source() - ref = source.placeholders.remove_placeholders(data) + ref = source.remove_placeholders(data) comment = TComment(ref, parser_pos=self.get_parser_pos()) self.append_child(comment) def handle_decl(self, decl: str) -> None: source = self.get_source() - ref = source.placeholders.remove_placeholders(decl) + ref = source.remove_placeholders(decl) if not ref.is_literal: raise ParsingError("Interpolations are not allowed in declarations.") elif decl.upper().startswith("DOCTYPE "): From 193f61355ddc77cad8a877d5dbb417c71c8c4ff3 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Tue, 30 Jun 2026 22:45:43 -0700 Subject: [PATCH 35/78] Use TemplateRef instead of str for starttag_text. --- tdom/parser.py | 32 ++++++++++++++------------------ tdom/processor.py | 4 +--- tdom/source.py | 2 +- 3 files changed, 16 insertions(+), 22 deletions(-) diff --git a/tdom/parser.py b/tdom/parser.py index 01fabe5d..4ac770a4 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -54,7 +54,7 @@ class OpenTagSourceInfo: they can contain embedded placeholders. """ - starttag_text: str + starttag_ref: TemplateRef " Entire starttag as parsed, includes placeholders, . " raw_attrs: tuple[HTMLAttribute, ...] " Attrs as parsed, includes placeholders. " @@ -65,7 +65,7 @@ class OpenTagSourceInfo: def close(self, endtag_pos: FrozenPosition | None = None) -> TagSourceInfo: return TagSourceInfo( - starttag_text=self.starttag_text, + starttag_ref=self.starttag_ref, raw_attrs=self.raw_attrs, startend=self.startend, starttag_pos=self.starttag_pos, @@ -284,7 +284,7 @@ def make_open_tag( tag=tag, attrs=self.make_tattrs(attrs), sinfo=OpenTagSourceInfo( - starttag_text=self.get_starttag_text(), + starttag_ref=self.get_starttag_ref(), raw_attrs=tuple(attrs), startend=startend, starttag_pos=parser_pos, @@ -316,12 +316,7 @@ def make_open_tag( # @NOTE: This must be called when the tag is handled since it is # populated based on the most recently finished start tag. Otherwise # the value will be out of sync. - starttag_text = self.get_starttag_text( - f"Expected startag_text to be set when parsing component at {i_index}." - ) - # Bypass placeholder tracking and just make ref directly. - # Placeholder tracking should be covered by the SourceTracker independently. - starttag_ref = source.placeholders.config.find_placeholders(starttag_text) + starttag_ref = self.get_starttag_ref() # @NOTE: The last string should terminate the starttag and end with ">" # So this length is the offset from the last interpolation to the start # of the children's leading string. @@ -336,7 +331,7 @@ def make_open_tag( attrs=self.make_tattrs(attrs), parser_pos=parser_pos, sinfo=OpenTagSourceInfo( - starttag_text=starttag_text, + starttag_ref=starttag_ref, raw_attrs=tuple(attrs), startend=startend, starttag_pos=parser_pos, @@ -460,9 +455,7 @@ def validate_end_tag(self, tag: str, open_tag: OpenTag) -> int | None: # current source slice and current line number # reader = source.get_reader() - starttag_ref = reader.placeholder_config.find_placeholders( - open_tag.sinfo.starttag_text - ) + starttag_ref = open_tag.sinfo.starttag_ref starttag_repr = reader.ref_to_repr(starttag_ref) endtag_repr = reader.ref_to_repr(tag_ref) endtag_pos_msg = reader.make_template_pos_msg(self.get_parser_pos()) @@ -496,16 +489,19 @@ def validate_end_tag(self, tag: str, open_tag: OpenTag) -> int | None: ) return tag_ref.i_indexes[0] - def get_starttag_text(self, msg: str = "Expecting starttag text to be set.") -> str: + def get_starttag_ref(self) -> TemplateRef: """ Wrap get_starttag_text and just raise if None is returned. Do this so we don't guard for `None` everywhere. """ - starttag_text = super().get_starttag_text() + starttag_text = self.get_starttag_text() if starttag_text is None: - raise ParsingAssertionError(msg) - return starttag_text + raise ParsingAssertionError( + "Expected the parser to have starttag_text set." + ) + # @NOTE: We assume the source tracker already manages the placeholders. + return self.get_source().find_placeholders(starttag_text) def has_ambiguous_forward_slash( self, sinfo: OpenTagSourceInfo | TagSourceInfo | None @@ -530,7 +526,7 @@ def has_ambiguous_forward_slash( # last char of last attr is "/" and sinfo.raw_attrs[-1][1][-1] == "/" # parsed starttag ends with "/>" - and sinfo.starttag_text.endswith("/>") + and sinfo.starttag_ref.strings[-1].endswith("/>") # if parsed as startend then its not ambiguous and not sinfo.startend ) diff --git a/tdom/processor.py b/tdom/processor.py index e21806a1..a6c385a4 100644 --- a/tdom/processor.py +++ b/tdom/processor.py @@ -788,9 +788,7 @@ def process( e_state.template, placeholder_config=e_state.ttree.placeholder_config, ) - starttag_repr = reader.ref_to_repr( - reader.placeholder_config.find_placeholders(sinfo.starttag_text) - ) + starttag_repr = reader.ref_to_repr(sinfo.starttag_ref) starttag_pos_msg = reader.make_template_pos_msg(sinfo.starttag_pos) e.add_note( f"Error occurred at {starttag_repr} at {starttag_pos_msg}." diff --git a/tdom/source.py b/tdom/source.py index ecafca9a..f1a8e8f0 100644 --- a/tdom/source.py +++ b/tdom/source.py @@ -39,7 +39,7 @@ class TagSourceInfo: they can contain embedded placeholders. """ - starttag_text: str + starttag_ref: TemplateRef " Entire starttag as parsed, includes placeholders, . " raw_attrs: tuple[HTMLAttribute, ...] " Attrs as parsed, includes placeholders. " From 0ef714663e6ef891fe04bb845746c4a093a60f2b Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Tue, 30 Jun 2026 23:00:51 -0700 Subject: [PATCH 36/78] Use TemplateRef for attrs in source info. --- tdom/parser.py | 30 ++++++++++++++++++++++-------- tdom/source.py | 3 +-- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/tdom/parser.py b/tdom/parser.py index 4ac770a4..5634accd 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -56,7 +56,7 @@ class OpenTagSourceInfo: starttag_ref: TemplateRef " Entire starttag as parsed, includes placeholders, . " - raw_attrs: tuple[HTMLAttribute, ...] + ref_attrs: tuple[tuple[TemplateRef, TemplateRef | None], ...] " Attrs as parsed, includes placeholders. " startend: bool " Was parsed as startend tag, ie. . " @@ -66,7 +66,7 @@ class OpenTagSourceInfo: def close(self, endtag_pos: FrozenPosition | None = None) -> TagSourceInfo: return TagSourceInfo( starttag_ref=self.starttag_ref, - raw_attrs=self.raw_attrs, + ref_attrs=self.ref_attrs, startend=self.startend, starttag_pos=self.starttag_pos, endtag_pos=endtag_pos, @@ -268,6 +268,20 @@ def make_tattrs(self, attrs: Sequence[HTMLAttribute]) -> tuple[TAttribute, ...]: """Build TAttributes from raw attribute tuples.""" return tuple(self.make_tattr(attr) for attr in attrs) + def make_ref_attr( + self, source: SourceTracker, attr: HTMLAttribute + ) -> tuple[TemplateRef, TemplateRef | None]: + return ( + source.find_placeholders(attr[0]), + source.find_placeholders(attr[1]) if attr[1] is not None else None, + ) + + def make_ref_attrs( + self, attrs: Sequence[HTMLAttribute] + ) -> tuple[tuple[TemplateRef, TemplateRef | None], ...]: + source = self.get_source() + return tuple(self.make_ref_attr(source, attr) for attr in attrs) + # ------------------------------------------ # Tag Helpers # ------------------------------------------ @@ -285,7 +299,7 @@ def make_open_tag( attrs=self.make_tattrs(attrs), sinfo=OpenTagSourceInfo( starttag_ref=self.get_starttag_ref(), - raw_attrs=tuple(attrs), + ref_attrs=self.make_ref_attrs(attrs), startend=startend, starttag_pos=parser_pos, ), @@ -332,7 +346,7 @@ def make_open_tag( parser_pos=parser_pos, sinfo=OpenTagSourceInfo( starttag_ref=starttag_ref, - raw_attrs=tuple(attrs), + ref_attrs=self.make_ref_attrs(attrs), startend=startend, starttag_pos=parser_pos, ), @@ -520,11 +534,11 @@ def has_ambiguous_forward_slash( if sinfo is not None: return ( # has attributes - len(sinfo.raw_attrs) > 0 + len(sinfo.ref_attrs) > 0 # last attr not bare attribute - and sinfo.raw_attrs[-1][1] is not None - # last char of last attr is "/" - and sinfo.raw_attrs[-1][1][-1] == "/" + and sinfo.ref_attrs[-1][1] is not None + # last char of last string of value of last ref attr is "/" + and sinfo.ref_attrs[-1][1].strings[-1][-1] == "/" # parsed starttag ends with "/>" and sinfo.starttag_ref.strings[-1].endswith("/>") # if parsed as startend then its not ambiguous diff --git a/tdom/source.py b/tdom/source.py index f1a8e8f0..bb5a0370 100644 --- a/tdom/source.py +++ b/tdom/source.py @@ -2,7 +2,6 @@ from dataclasses import dataclass from string.templatelib import Interpolation, Template -from .parser_utils import HTMLAttribute from .placeholders import PlaceholderConfig from .template_utils import TemplateRef @@ -41,7 +40,7 @@ class TagSourceInfo: starttag_ref: TemplateRef " Entire starttag as parsed, includes placeholders, . " - raw_attrs: tuple[HTMLAttribute, ...] + ref_attrs: tuple[tuple[TemplateRef, TemplateRef | None], ...] " Attrs as parsed, includes placeholders. " startend: bool " Was parsed as startend tag, ie. . " From 48dfdf13c01d1d887aaf4407e7f93957b625d0b1 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Wed, 1 Jul 2026 22:05:13 -0700 Subject: [PATCH 37/78] Rein in placeholders with placeholder-independent PartPosition. --- tdom/parser.py | 136 +++++++++---------- tdom/parser_test.py | 4 +- tdom/parser_utils.py | 107 +++++++++++++++ tdom/processor.py | 22 +--- tdom/source.py | 291 ++++++----------------------------------- tdom/template_utils.py | 85 +++++++----- tdom/tnodes.py | 41 ++++-- 7 files changed, 305 insertions(+), 381 deletions(-) diff --git a/tdom/parser.py b/tdom/parser.py index 5634accd..90de8472 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -4,19 +4,18 @@ from string.templatelib import Interpolation, Template from .htmlspec import VOID_ELEMENTS -from .parser_utils import HTMLAttribute +from .parser_utils import HTMLAttribute, parser_pos_to_part_pos from .placeholders import ( - PlaceholderConfig, PlaceholderState, - make_placeholder_config, ) from .source import ( - FrozenPosition, + LinePosition, + PartPosition, SourceReader, - TagSourceInfo, ) from .template_utils import TemplateRef, combine_template_refs from .tnodes import ( + TagSourceInfo, TAttribute, TComment, TComponent, @@ -48,22 +47,24 @@ class AttributeParsingError(ParsingError): @dataclass(frozen=True, slots=True) class OpenTagSourceInfo: """ - Retained tag information from the parsed source. + Retained tag information from the parsed source meant for error reporting. - @NOTE: These properties DEPEND on the placeholder configuration because - they can contain embedded placeholders. + @NOTE: This is an temporary structure that will be finalized when the + tag is closed. + + @TODO: Do we need `ref_attrs` or should we just try to get by with the tattrs? """ starttag_ref: TemplateRef - " Entire starttag as parsed, includes placeholders, . " + " Entire starttag as parsed except placeholders are replaced by references. " ref_attrs: tuple[tuple[TemplateRef, TemplateRef | None], ...] - " Attrs as parsed, includes placeholders. " + " Attrs as parsed except placeholders are replaced by references. " startend: bool " Was parsed as startend tag, ie. . " - starttag_pos: FrozenPosition - " Position of the parser when the element starttag was parsed. " + starttag_pos: PartPosition + " Template part position of the starttag. " - def close(self, endtag_pos: FrozenPosition | None = None) -> TagSourceInfo: + def close(self, endtag_pos: PartPosition | None = None) -> TagSourceInfo: return TagSourceInfo( starttag_ref=self.starttag_ref, ref_attrs=self.ref_attrs, @@ -77,14 +78,14 @@ def close(self, endtag_pos: FrozenPosition | None = None) -> TagSourceInfo: class OpenTElement: tag: str attrs: tuple[TAttribute, ...] - parser_pos: FrozenPosition + source_pos: PartPosition sinfo: OpenTagSourceInfo children: list[TNode] = field(default_factory=list) @dataclass class OpenTFragment: - parser_pos: FrozenPosition | None = None + source_pos: PartPosition | None = None children: list[TNode] = field(default_factory=list) @@ -96,7 +97,7 @@ class OpenTComponent: offset_into_children_start_s: int """The offset INTO the starting string where the component's children template starts.""" attrs: tuple[TAttribute, ...] - parser_pos: FrozenPosition + source_pos: PartPosition sinfo: OpenTagSourceInfo # @NOTE: The `children` are discarded after parsing and are just used to # track template consistency or assist with error reporting. If the @@ -117,7 +118,7 @@ class SourceTracker: template: Template - placeholders: PlaceholderState + placeholders: PlaceholderState = field(default_factory=lambda: PlaceholderState()) # if i_index >= s_index, feeding an interpolation; # otherwise, when i_index < s_index, feeding a string. @@ -169,9 +170,7 @@ def format_endtag(self, i_index: int) -> str: return self.get_expression(i_index, fallback_prefix="component-endtag") def get_reader(self) -> SourceReader: - return SourceReader( - template=self.template, placeholder_config=self.placeholders.config - ) + return SourceReader(template=self.template) def remove_placeholders(self, text: str) -> TemplateRef: """ @@ -190,6 +189,14 @@ def find_placeholders(self, text: str) -> TemplateRef: """ return self.placeholders.config.find_placeholders(text) + def translate_pos(self, parser_pos: LinePosition) -> PartPosition: + """ + Translate the parser position into a part position in the template. + """ + return parser_pos_to_part_pos( + self.template, self.placeholders.config, parser_pos + ) + class TemplateParser(HTMLParser): root: OpenTFragment @@ -198,7 +205,7 @@ class TemplateParser(HTMLParser): " Map from completed tnodes back to their opentag for error reporting. " tcomponent_children: dict[TComponent, list[TNode]] "List of children for each finished tcomponent, stored at closing. " - sinfo_table: dict[FrozenPosition, TagSourceInfo] + sinfo_table: dict[PartPosition, TagSourceInfo] " Tags with more source info than just a position are tracked in this mapping. " def __init__(self, *, convert_charrefs: bool = True): @@ -217,7 +224,7 @@ def append_child(self, child: TNode) -> None: parent = self.get_parent() parent.children.append(child) - def get_parser_pos(self) -> FrozenPosition: + def get_parser_pos(self) -> LinePosition: """ Get the current position of the parser. @@ -227,7 +234,13 @@ def get_parser_pos(self) -> FrozenPosition: unique across a "family" of templates with the same structure. """ line, offset = self.getpos() - return FrozenPosition(line=line, offset=offset) + return LinePosition(line=line, offset=offset) + + def get_source_pos(self, parser_pos: LinePosition | None = None) -> PartPosition: + source = self.get_source() + return source.translate_pos( + self.get_parser_pos() if parser_pos is None else parser_pos + ) # ------------------------------------------ # Attribute Helpers @@ -293,7 +306,7 @@ def make_open_tag( source = self.get_source() tag_ref = source.placeholders.remove_placeholders(tag) if tag_ref.is_literal: - parser_pos = self.get_parser_pos() + source_pos = self.get_source_pos() open_tag = OpenTElement( tag=tag, attrs=self.make_tattrs(attrs), @@ -301,9 +314,9 @@ def make_open_tag( starttag_ref=self.get_starttag_ref(), ref_attrs=self.make_ref_attrs(attrs), startend=startend, - starttag_pos=parser_pos, + starttag_pos=source_pos, ), - parser_pos=parser_pos, + source_pos=source_pos, ) return open_tag @@ -336,19 +349,19 @@ def make_open_tag( # of the children's leading string. offset_into_children_start_s = len(starttag_ref.strings[-1]) - parser_pos = self.get_parser_pos() + source_pos = self.get_source_pos() open_tag = OpenTComponent( start_i_index=i_index, children_start_s_index=children_start_s_index, offset_into_children_start_s=offset_into_children_start_s, attrs=self.make_tattrs(attrs), - parser_pos=parser_pos, + source_pos=source_pos, sinfo=OpenTagSourceInfo( starttag_ref=starttag_ref, ref_attrs=self.make_ref_attrs(attrs), startend=startend, - starttag_pos=parser_pos, + starttag_pos=source_pos, ), ) return open_tag @@ -357,7 +370,7 @@ def finalize_tag( self, open_tag: OpenTag, endtag_i_index: int | None = None, - endtag_pos: FrozenPosition | None = None, + endtag_pos: PartPosition | None = None, ) -> TNode: """Finalize an OpenTag into a TNode.""" source = self.get_source() @@ -366,24 +379,24 @@ def finalize_tag( tag=tag, attrs=attrs, children=children, - parser_pos=parser_pos, + source_pos=source_pos, sinfo=sinfo, ): tnode = TElement( tag=tag, attrs=attrs, children=tuple(children), - parser_pos=parser_pos, + source_pos=source_pos, ) - self.sinfo_table[parser_pos] = sinfo.close(endtag_pos=endtag_pos) - case OpenTFragment(children=children, parser_pos=parser_pos): - tnode = TFragment(children=tuple(children), parser_pos=parser_pos) + self.sinfo_table[source_pos] = sinfo.close(endtag_pos=endtag_pos) + case OpenTFragment(children=children, source_pos=source_pos): + tnode = TFragment(children=tuple(children), source_pos=source_pos) case OpenTComponent( start_i_index=start_i_index, children_start_s_index=children_start_s_index, offset_into_children_start_s=offset_into_children_start_s, attrs=attrs, - parser_pos=parser_pos, + source_pos=source_pos, sinfo=sinfo, children=children, ): @@ -399,9 +412,9 @@ def finalize_tag( end_i_index=endtag_i_index, children_ref=children_ref, attrs=attrs, - parser_pos=parser_pos, + source_pos=source_pos, ) - self.sinfo_table[parser_pos] = sinfo.close(endtag_pos=endtag_pos) + self.sinfo_table[source_pos] = sinfo.close(endtag_pos=endtag_pos) self.tcomponent_children[tnode] = children return tnode @@ -472,8 +485,8 @@ def validate_end_tag(self, tag: str, open_tag: OpenTag) -> int | None: starttag_ref = open_tag.sinfo.starttag_ref starttag_repr = reader.ref_to_repr(starttag_ref) endtag_repr = reader.ref_to_repr(tag_ref) - endtag_pos_msg = reader.make_template_pos_msg(self.get_parser_pos()) - starttag_pos_msg = reader.make_template_pos_msg(open_tag.parser_pos) + endtag_pos_msg = reader.make_template_pos_msg(self.get_source_pos()) + starttag_pos_msg = reader.make_template_pos_msg(open_tag.source_pos) raise ParsingError( f"Component closing tag at {endtag_pos_msg} found for element {starttag_repr} at {starttag_pos_msg}." ) @@ -571,7 +584,7 @@ def handle_endtag(self, tag: str) -> None: if tag_ref.is_literal: # EXAMPLE 1: getting line number, does not slice/extract source reader = source.get_reader() - pos_msg = reader.make_template_pos_msg(self.get_parser_pos()) + pos_msg = reader.make_template_pos_msg(self.get_source_pos()) raise ParsingError( f"Unexpected closing tag with no open tag, {pos_msg}." ) @@ -583,7 +596,7 @@ def handle_endtag(self, tag: str) -> None: # Component tag endtag but no component tag is open... # EXAMPLE 2: getting line number AND get source (repr)esentation via ref reader = source.get_reader() - pos_msg = reader.make_template_pos_msg(self.get_parser_pos()) + pos_msg = reader.make_template_pos_msg(self.get_source_pos()) tag_repr = reader.ref_to_repr(tag_ref) raise ParsingError( f"Unexpected closing component tag with no open tag, {pos_msg}." @@ -591,7 +604,9 @@ def handle_endtag(self, tag: str) -> None: open_tag = self.stack.pop() endtag_i_index = self.validate_end_tag(tag, open_tag) final_tag = self.finalize_tag( - open_tag, endtag_i_index=endtag_i_index, endtag_pos=self.get_parser_pos() + open_tag, + endtag_i_index=endtag_i_index, + endtag_pos=self.get_source_pos(), ) self.append_child(final_tag) @@ -636,15 +651,15 @@ def handle_data(self, data: str) -> None: parent.children[-1] = TText( ref=combine_template_refs(prior_text.ref, ref), # Keep starting position of the prior text - parser_pos=prior_text.parser_pos, + source_pos=prior_text.source_pos, ) else: - self.append_child(TText(ref=ref, parser_pos=self.get_parser_pos())) + self.append_child(TText(ref=ref, source_pos=self.get_source_pos())) def handle_comment(self, data: str) -> None: source = self.get_source() ref = source.remove_placeholders(data) - comment = TComment(ref, parser_pos=self.get_parser_pos()) + comment = TComment(ref, source_pos=self.get_source_pos()) self.append_child(comment) def handle_decl(self, decl: str) -> None: @@ -654,7 +669,7 @@ def handle_decl(self, decl: str) -> None: raise ParsingError("Interpolations are not allowed in declarations.") elif decl.upper().startswith("DOCTYPE "): doctype_content = decl[7:].strip() - doctype = TDocumentType(doctype_content, parser_pos=self.get_parser_pos()) + doctype = TDocumentType(doctype_content, source_pos=self.get_source_pos()) self.append_child(doctype) else: raise ParsingError( @@ -710,8 +725,8 @@ def close(self) -> None: ) ): sinfo = ( - self.sinfo_table.get(comp.parser_pos) - if comp.parser_pos is not None + self.sinfo_table.get(comp.source_pos) + if comp.source_pos is not None else None ) starttag = source.format_starttag(comp.start_i_index) @@ -755,7 +770,6 @@ def get_tnode(self) -> TNode: def get_ttree(self) -> TTree: return TTree( self.get_tnode(), - placeholder_config=self.get_source().placeholders.config, sinfos=tuple(self.sinfo_table.values()), ) @@ -769,21 +783,15 @@ def get_source(self) -> SourceTracker: raise AssertionError("Source has not been initialized.") return self.source - def feed_template( - self, template: Template, placeholder_config: PlaceholderConfig - ) -> None: + def feed_template(self, template: Template) -> None: """Feed a Template's content to the parser.""" assert self.source is None, "Did you forget to call reset?" - self.source = SourceTracker( - template, placeholders=PlaceholderState(config=placeholder_config) - ) + self.source = SourceTracker(template) for content in self.source: self.feed(content) @staticmethod - def parse( - t: Template, placeholder_config: PlaceholderConfig | None = None - ) -> TNode: + def parse(t: Template) -> TNode: """ Parse a Template containing valid HTML and substitutions and return a cacheable TNode tree representing its structure. @@ -791,15 +799,11 @@ def parse( A placeholder config must be passed to keep parser positions consistent between calls. """ - return TemplateParser.parse_to_ttree(t, placeholder_config).root + return TemplateParser.parse_to_ttree(t).root @staticmethod - def parse_to_ttree( - t: Template, placeholder_config: PlaceholderConfig | None = None - ) -> TTree: - if placeholder_config is None: - placeholder_config = make_placeholder_config() + def parse_to_ttree(t: Template) -> TTree: parser = TemplateParser() - parser.feed_template(t, placeholder_config=placeholder_config) + parser.feed_template(t) parser.close() return parser.get_ttree() diff --git a/tdom/parser_test.py b/tdom/parser_test.py index a93e9c01..27c1945f 100644 --- a/tdom/parser_test.py +++ b/tdom/parser_test.py @@ -8,7 +8,7 @@ TemplateParser, ) from .placeholders import make_placeholder_config -from .source import FrozenPosition +from .source import PartPosition from .template_utils import TemplateRef from .tnodes import ( TComment, @@ -704,4 +704,4 @@ def test_tnode_parser_position(): ) el = tnode.children[0] assert isinstance(el, tnode_type) - assert el.parser_pos == FrozenPosition(line=1, offset=len("
")) + assert el.source_pos == PartPosition.pack_s_index(0, offset=len("
")) diff --git a/tdom/parser_utils.py b/tdom/parser_utils.py index f8365871..f2c0394a 100644 --- a/tdom/parser_utils.py +++ b/tdom/parser_utils.py @@ -1 +1,108 @@ +from string.templatelib import Template + +from .placeholders import PlaceholderConfig +from .source import LinePosition, MutableLinePosition +from .template_utils import PartPosition + type HTMLAttribute = tuple[str, str | None] + + +def parser_pos_to_part_pos( + template: Template, + placeholder_config: PlaceholderConfig, + parser_pos: LinePosition, +) -> PartPosition: + """ + Translate the given parser position into a template part position. + """ + pos = MutableLinePosition() + combined_size = 2 * len(template.strings) - 1 + last_index = combined_size - 1 + for index in range(combined_size): + if index % 2 == 0: + s = template.strings[index // 2] + if parser_pos.line > pos.line: + # need more lines + nls_found = s.count("\n") # how many were found? + nls_need = parser_pos.line - pos.line # how many are needed? + if nls_found >= nls_need: + pos.line += nls_need + offset_found = len(s.split("\n", nls_need + 1)[nls_need]) + if offset_found >= parser_pos.offset: + # needed lines, found lines, found offset + pos.offset = parser_pos.offset + total_offset = ( + sum( + len(line) + 1 + for line in s.split("\n", nls_need + 1)[:nls_need] + ) + + parser_pos.offset + ) + return PartPosition(index, total_offset) + else: + # got enough lines, still need more offset + pos.offset = offset_found + elif nls_found > 0: + # some lines but still need more lines + pos.line += nls_found + pos.offset = len(s[s.rfind("\n") + 1 :]) + else: + # no lines, still need more lines + offset_found = len(s) + pos.offset += offset_found + elif parser_pos.line == pos.line: + # got enough lines, we just need more offset + offset_found = len(s[: s.find("\n")]) if "\n" in s else len(s) + offset_need = parser_pos.offset - pos.offset + if offset_found > offset_need: + pos.offset += offset_need + total_offset = offset_need # only from the start of this string. + # had lines, found offset + return PartPosition(index, total_offset) + elif offset_found == offset_need: + if index < last_index: + # @TODO: Start at the interpolation ? + return PartPosition(index + 1, 0) + else: + # @TEST + # @TODO: Is this possible? Seems like this position would + # technically be undefined and an error. + # Start at the very end of the last part (can this exist?) + return PartPosition(index, len(s)) + else: + pos.offset += offset_found + else: + # We should have dropped out and failed earlier this would be a bug. + raise AssertionError( + f"Unexpected line: {pos.line} greater than asked for {parser_pos.line}" + ) + + else: + i_index = (index - 1) // 2 + ph_length = len(placeholder_config.make_placeholder(i_index)) + if ( + pos.line == parser_pos.line + and pos.offset + ph_length > parser_pos.offset + ): + # Ie. we don't know how to determine how much of the + # interpolation expression would be equivalent to + # a substring of a placeholder. + raise ValueError( + f"Cannot split a placeholder for interpolations[{i_index}], placeholders are atomic." + ) + pos.offset += ph_length + if pos == parser_pos: + # An offset to the end of this interpolation should be the start + # of the following string. + # @TEST + # @TODO: Do we need this check or would it be picked up + # in the next iteration? + return PartPosition(index + 1, 0) + if pos == parser_pos: + # @TEST + # @TODO: When can this fall through happen? Or is this always an error? + return PartPosition(last_index, len(template.strings[-1])) + else: + raise ValueError( + "Unexpected position {pos}, did not reach required position {parser_pos}" + ) diff --git a/tdom/processor.py b/tdom/processor.py index a6c385a4..0a2ae931 100644 --- a/tdom/processor.py +++ b/tdom/processor.py @@ -31,7 +31,6 @@ ) from .parser import ParsingError, TemplateParser from .parser_utils import HTMLAttribute -from .placeholders import PlaceholderConfig, make_placeholder_config from .protocols import HasHTMLDunder from .scope import ScopedTemplate from .source import SourceReader @@ -581,19 +580,11 @@ def to_ttree(self, template: Template) -> TTree: ... @dataclass(frozen=True) class TemplateParserProxy(ITemplateParserProxy): - placeholder_config: PlaceholderConfig = field( - default_factory=make_placeholder_config - ) - def to_tnode(self, template: Template) -> TNode: # BWC - return TemplateParser.parse( - template, placeholder_config=self.placeholder_config - ) + return TemplateParser.parse(template) def to_ttree(self, template: Template) -> TTree: - return TemplateParser.parse_to_ttree( - template, placeholder_config=self.placeholder_config - ) + return TemplateParser.parse_to_ttree(template) @dataclass(frozen=True) @@ -772,21 +763,20 @@ def process( # even get started because the template wouldn't parse. continue sinfo_table = e_state.ttree.unpack_sinfo_table() - sinfo = parser_pos = None + sinfo = source_pos = None if isinstance( e_state.tnode, (TElement, TText, TComment, TComponent, TDocumentType), ): - parser_pos = e_state.tnode.parser_pos - if parser_pos: - sinfo = sinfo_table.get(parser_pos, None) + source_pos = e_state.tnode.source_pos + if source_pos: + sinfo = sinfo_table.get(source_pos, None) if sinfo: # # Example 4: Getting starttag repr and pos in processor. # reader = SourceReader( e_state.template, - placeholder_config=e_state.ttree.placeholder_config, ) starttag_repr = reader.ref_to_repr(sinfo.starttag_ref) starttag_pos_msg = reader.make_template_pos_msg(sinfo.starttag_pos) diff --git a/tdom/source.py b/tdom/source.py index bb5a0370..79840b24 100644 --- a/tdom/source.py +++ b/tdom/source.py @@ -2,12 +2,11 @@ from dataclasses import dataclass from string.templatelib import Interpolation, Template -from .placeholders import PlaceholderConfig -from .template_utils import TemplateRef +from .template_utils import PartPosition, TemplateRef, slice_from_template @dataclass(slots=True, frozen=True) -class FrozenPosition: +class LinePosition: "A immutable position in a block of source code." line: int = 1 @@ -17,79 +16,26 @@ class FrozenPosition: @dataclass(slots=True) -class Position: - "A position in a block of source code." +class MutableLinePosition: + "A mutable position in a block of source code." line: int = 1 " Line of code, starts at 1. " offset: int = 0 " Offset from the start of the line, starts at 0. " - def freeze(self) -> FrozenPosition: - return FrozenPosition(line=self.line, offset=self.offset) + def freeze(self) -> LinePosition: + "Freeze ourself into an immutable object with the same values." + return LinePosition(line=self.line, offset=self.offset) -@dataclass(frozen=True, slots=True) -class TagSourceInfo: - """ - Retained tag information from the parsed source. - - @NOTE: These properties DEPEND on the placeholder configuration because - they can contain embedded placeholders. +def template_repr_iter(template: Template) -> t.Generator[str]: """ + Yield a string representation of each part of a given template. - starttag_ref: TemplateRef - " Entire starttag as parsed, includes placeholders, . " - ref_attrs: tuple[tuple[TemplateRef, TemplateRef | None], ...] - " Attrs as parsed, includes placeholders. " - startend: bool - " Was parsed as startend tag, ie. . " - starttag_pos: FrozenPosition - " Position of the parser when the element starttag was parsed. " - endtag_pos: FrozenPosition | None = None - " Position of the parser when the element endtag was parsed. " - - -@dataclass(frozen=True) -class MultiPosition: + @NOTE: This will not yield empty strings because it uses the underlying + template iterator which does not. """ - @NOTE: Like other position tools this does not support positioning within - an interpolation. Either you are at the start of interpolation or at an - offset within a string. - """ - - pos: FrozenPosition - tpos: FrozenPosition - s_index: int - s_offset: int - i_index: int - - -def iterate_template_from_s_index(template: Template, s_index: int): - last_index = len(template.strings) - 1 - if s_index < 0 or s_index > last_index: - return - index = 0 - while index <= last_index: - yield template.strings[index] - if index < last_index: - yield template.interpolations[index] - index += 1 - - -def iterate_template_from_i_index(template: Template, i_index: int): - last_index = len(template.strings) - 1 - if i_index < 0 or i_index >= last_index: - return - index = 0 - while index <= last_index: - if index < last_index: - yield template.interpolations[index] - index += 1 - yield template.strings[index] - - -def template_repr_iter(template: Template) -> t.Generator[str]: for part in template: if isinstance(part, str): yield part @@ -98,10 +44,16 @@ def template_repr_iter(template: Template) -> t.Generator[str]: def template_repr(template: Template) -> str: + """ + Create a string representation of the given template. + """ return "".join(template_repr_iter(template)) def interpolation_repr(ip: Interpolation) -> str: + """ + Create a string representation of the given interpolation. + """ expr_str = ip.expression conversion_str = f"!{ip.conversion}" if ip.conversion is not None else "" format_spec_str = f":{ip.format_spec}" if ip.format_spec else "" @@ -110,198 +62,39 @@ def interpolation_repr(ip: Interpolation) -> str: @dataclass class SourceReader: - "Format report-like strings from template source for the parser." + "Format report-like strings from template source for error reporting." template: Template - placeholder_config: PlaceholderConfig - def ref_to_repr(self, ref: TemplateRef, limit: int | None = None) -> str: + """ + Convert tref to string representation of the underlying template. + """ filled_template = ref.resolve(self.template.interpolations) return template_repr(filled_template)[:limit] - def make_template_pos_msg(self, parser_pos: FrozenPosition) -> str: - template_pos = self.to_template_pos(parser_pos) + def make_template_pos_msg(self, source_pos: PartPosition) -> str: + """ + Make a message to display the line number and offset number. + """ + template_pos = self.to_template_pos(source_pos) return f"line {template_pos.line} offset {template_pos.offset}" - def to_template_pos(self, parser_pos: FrozenPosition) -> FrozenPosition: - mpos = self._compute_positions(parser_pos) - return mpos.tpos - - def make_template_slice(self, parser_pos: FrozenPosition, limit: int | None = None): - mpos = self._compute_positions(parser_pos) - if mpos.s_index > mpos.i_index: - sub_template = Template( - *iterate_template_from_s_index(self.template, mpos.s_index) - ) - return template_repr(sub_template)[mpos.s_offset : limit] - else: - sub_template = Template( - *iterate_template_from_i_index(self.template, mpos.i_index) - ) - return template_repr(sub_template)[:limit] - - def _compute_positions(self, parser_pos: FrozenPosition) -> MultiPosition: - return compute_template_positions( - self.template, self.placeholder_config, parser_pos - ) - - -def compute_template_positions( - template: Template, - placeholder_config: PlaceholderConfig, - parser_pos: FrozenPosition, -) -> MultiPosition: - """ - Translate the given parser (pos)ition into template (pos)ition. - - @NOTE: There can be newlines in an interpolation expression which - results in the parser position's line being less than the - template position's line since a placeholder will not contain newlines. - - @NOTE: Similarly an interpolation as displayed can be longer than a - placeholder OR shorter than a placeholder causing the offsets to go - out of sync. - - @NOTE: There is a weird issue with `format_spec` and the specification - where you can't tell if a ':' was used or not when the `format_spec` is - empty. We just assume no one would leave it in without a non-empty - format_spec, ie. t"{val:}" would not exist even though it is valid. The - conversion does not have this issue because "{val!}" is invalid and when - no conversion is set the conversion value is None. - """ - # - # Walk until we reach the given parser pos, keeping both parser position - # and template position in sync. When the given parser position is reached - # then return the synced up template position. - # - pos = Position() - tpos = Position() - last_s_index = len(template.strings) - 1 - for s_index in range(len(template.strings)): - # - # Walk through `strings[s_index]` - # - s = template.strings[s_index] - if parser_pos.line > pos.line: - # need more lines - nls_found = s.count("\n") # how many were found? - nls_need = parser_pos.line - pos.line # how many are needed? - if nls_found >= nls_need: - pos.line += nls_need - tpos.line += nls_need - offset_found = len(s.split("\n", nls_need + 1)[nls_need]) - if offset_found >= parser_pos.offset: - # needed lines, found lines, found offset - tpos.offset = pos.offset = parser_pos.offset - total_offset = ( - sum( - len(line) + 1 - for line in s.split("\n", nls_need + 1)[:nls_need] - ) - + parser_pos.offset - ) - return MultiPosition( - pos=parser_pos, - tpos=tpos.freeze(), - s_index=s_index, - i_index=s_index - 1, - s_offset=total_offset, - ) - else: - # got enough lines, still need more offset - tpos.offset = pos.offset = offset_found - elif nls_found > 0: - # some lines but still need more lines - pos.line += nls_found - tpos.line += nls_found - tpos.offset = pos.offset = len(s[s.rfind("\n") + 1 :]) + def to_template_pos(self, source_pos: PartPosition) -> LinePosition: + """ + Convert a (template) part position into a line position based on the + string representation of the template. + """ + pos = MutableLinePosition() + for part in slice_from_template(self.template, start=None, stop=source_pos): + if isinstance(part, str): + text = part else: - # no lines, still need more lines - offset_found = len(s) - tpos.offset += offset_found - pos.offset += offset_found - elif parser_pos.line == pos.line: - # got enough lines, we just need more offset - offset_found = len(s[: s.find("\n")]) if "\n" in s else len(s) - offset_need = parser_pos.offset - pos.offset - if offset_found >= offset_need: - pos.offset += offset_need - tpos.offset += offset_need - total_offset = offset_need # only from the start of this string. - # had lines, found offset - return MultiPosition( - pos=parser_pos, - tpos=tpos.freeze(), - s_index=s_index, - i_index=s_index - 1, - s_offset=total_offset, - ) - else: - tpos.offset += offset_found - pos.offset += offset_found - else: - # We should have dropped out and failed earlier this would be a bug. - raise AssertionError( - f"Unexpected line: {pos.line} greater than asked for {parser_pos.line}" - ) - - # - # Walk through `interpolations[s_index]` - # - if s_index < last_s_index: - ph_length = len(placeholder_config.make_placeholder(s_index)) - if ( - pos.line == parser_pos.line - and pos.offset + ph_length > parser_pos.offset - ): - # Ie. we don't know how to determine how much of the - # interpolation expression would be equivalent to - # a substring of a placeholder. - raise ValueError( - f"Cannot split a placeholder for interpolations[{s_index}], placeholders are atomic." - ) - - ip = template.interpolations[s_index] - expr = ip.expression - expr_line_count = expr.count("\n") - tpos.line += expr_line_count - pos.offset += ph_length - EXCLAIMATION_POINT = CONVERSION_CHAR = SEMICOLON = LEFT_CURLY_BRACE = ( - RIGHT_CURLY_BRACE - ) = 1 - tail = ( - ( - EXCLAIMATION_POINT + CONVERSION_CHAR - if ip.conversion is not None - else 0 - ) # "!" and conversion char or neither - + (SEMICOLON if ip.format_spec else 0) # ":" or not - + len(ip.format_spec) - + RIGHT_CURLY_BRACE - ) - if expr_line_count > 0: - tpos.offset = len(expr[expr.rfind("\n") + 1 :]) + tail + text = interpolation_repr(part) + nls = text.count("\n") + if nls: + pos.offset = len(text) - (text.rfind("\n") + 1) + pos.line += nls else: - tpos.offset += LEFT_CURLY_BRACE + len(expr) + tail - if pos == parser_pos: - return MultiPosition( - pos=parser_pos, - tpos=tpos.freeze(), - s_index=s_index, - i_index=s_index, - s_offset=0, - ) - if pos == parser_pos: - # @TODO: When can this fall through happen? Or is this always an error? - return MultiPosition( - pos=parser_pos, - tpos=tpos.freeze(), - s_index=len(template.strings) - 1, - i_index=len(template.strings) - 2, - s_offset=len(template.strings[-1]), - ) - else: - raise ValueError( - "Unexpected position {pos}, did not reach required position {parser_pos}" - ) + pos.offset += len(text) + return pos.freeze() diff --git a/tdom/template_utils.py b/tdom/template_utils.py index 096d3351..ad3ea20d 100644 --- a/tdom/template_utils.py +++ b/tdom/template_utils.py @@ -94,56 +94,69 @@ def resolve(self, interpolations: tuple[Interpolation, ...]) -> Template: def slice_from_template( - template: Template, tslice: TemplateSlice + template: Template, + start: PartPosition | None = None, + stop: PartPosition | None = None, ) -> t.Generator[Interpolation | str]: """ Yield the template parts that make up the requested slice. """ - if tslice.start is None: - first = 0 - else: - first = tslice.start - assert first >= 0 and first < len(template.strings) - if tslice.start_offset is None: - offset = None - else: - offset = tslice.start_offset - if tslice.stop is None: - last = len(template.strings) - 1 - else: - last = tslice.stop - 1 - assert last >= 0 and last < len(template.strings) - if tslice.stop_limit is None: - limit = None - else: - limit = tslice.stop_limit + first = start.index if start and start.index is not None else 0 + offset = start.offset if start else None + last = ( + stop.index if stop and stop.index is not None else 2 * len(template.strings) - 1 + ) + limit = stop.offset if stop else None if first == last: - yield template.strings[first][offset:limit] + if first % 2 == 0: + yield template.strings[first][offset:limit] + else: + # @NOTE: No offset OR limit applied to interpolations. + yield template.interpolations[first] return else: - yield template.strings[first][offset:] - yield template.interpolations[first] + if first % 2 == 0: + yield template.strings[first // 2][offset:] + else: + # @NOTE: No offset applied to interpolations. + yield template.interpolations[(first - 1) // 2] for index in range(first + 1, last + 1): - if index == last: - yield template.strings[last][:limit] + if index % 2 == 0: + if index == last: + yield template.strings[index // 2][:limit] + else: + yield template.strings[index // 2] else: - yield template.strings[index] - yield template.interpolations[index] + # @NOTE: No limit applied to interpolations. + yield template.interpolations[(index - 1) // 2] -@dataclass(frozen=True, slots=True) -class TemplateSlice: +@dataclass(slots=True, frozen=True) +class PartPosition: """ - strings[start][start_offset:] - ... - strings[stop][:stop_limit] + A template part position. + + Translate indexes into strings by multiplying by 2. + ie. 0->0, 1->2, 2->4, etc. + Reverse by dividing by 2. - @NOTE: Start offset could be len(string[start]) and likewise stop_limit could be 0. + Translate indexes into interpolations by multiplying by 2 and then adding 1. + ie. 0->1, 1->3, 2->5, etc. + Reverse by subtracting 1 and dividing by 2. """ - start: int | None = None - start_offset: int | None = None - stop: int | None = None - stop_limit: int | None = None + index: int + " Index of the template parts, translate for strings/interpolations. " + + offset: int = 0 + " Offset from the start of the template part. " + + @classmethod + def pack_s_index(cls, s_index: int, offset: int = 0): + return cls(index=s_index * 2, offset=offset) + + @classmethod + def pack_i_index(cls, i_index: int, offset: int = 0): + return cls(index=i_index * 2 + 1, offset=offset) diff --git a/tdom/tnodes.py b/tdom/tnodes.py index 935d86b6..1b375e8b 100644 --- a/tdom/tnodes.py +++ b/tdom/tnodes.py @@ -1,9 +1,7 @@ import typing as t from dataclasses import dataclass, field -from .placeholders import PlaceholderConfig -from .source import FrozenPosition, TagSourceInfo -from .template_utils import TemplateRef +from .template_utils import PartPosition, TemplateRef @dataclass(slots=True, frozen=True) @@ -47,7 +45,7 @@ def __str__(self) -> str: class TText(TNode): ref: TemplateRef - parser_pos: FrozenPosition | None = field(default=None, compare=False) + source_pos: PartPosition | None = field(default=None, compare=False) @classmethod def empty(cls) -> t.Self: @@ -62,7 +60,7 @@ def literal(cls, text: str) -> t.Self: class TComment(TNode): ref: TemplateRef - parser_pos: FrozenPosition | None = field(default=None, compare=False) + source_pos: PartPosition | None = field(default=None, compare=False) @classmethod def literal(cls, text: str) -> t.Self: @@ -73,14 +71,14 @@ def literal(cls, text: str) -> t.Self: class TDocumentType(TNode): text: str - parser_pos: FrozenPosition | None = field(default=None, compare=False) + source_pos: PartPosition | None = field(default=None, compare=False) @dataclass(slots=True, frozen=True) class TFragment(TNode): children: tuple[TNode, ...] = field(default_factory=tuple) - parser_pos: FrozenPosition | None = field(default=None, compare=False) + source_pos: PartPosition | None = field(default=None, compare=False) @dataclass(slots=True, frozen=True) @@ -89,7 +87,7 @@ class TElement(TNode): attrs: tuple[TAttribute, ...] = field(default_factory=tuple) children: tuple[TNode, ...] = field(default_factory=tuple) - parser_pos: FrozenPosition | None = field(default=None, compare=False) + source_pos: PartPosition | None = field(default=None, compare=False) @dataclass(slots=True, frozen=True) @@ -107,18 +105,37 @@ class TComponent(TNode): attrs: tuple[TAttribute, ...] = field(default_factory=tuple) - parser_pos: FrozenPosition | None = field(default=None, compare=False) + source_pos: PartPosition | None = field(default=None, compare=False) + + +@dataclass(frozen=True, slots=True) +class TagSourceInfo: + """ + Retained tag information from the parsed source meant for error reporting. + + @NOTE: This must be cacheable so it should not directly reference a + template instance. + """ + + starttag_ref: TemplateRef + " Entire starttag as parsed except placeholders are replaced by references. " + ref_attrs: tuple[tuple[TemplateRef, TemplateRef | None], ...] + " Attrs as parsed except placeholders are replaced by references. " + startend: bool + " Was parsed as startend tag, ie. . " + starttag_pos: PartPosition + " Template part position of the starttag, ie. or . " + endtag_pos: PartPosition | None = None + " Template part position of the endtag, ie. . " @dataclass class TTree: root: TNode - placeholder_config: PlaceholderConfig - sinfos: tuple[TagSourceInfo, ...] = () - def unpack_sinfo_table(self) -> dict[FrozenPosition, TagSourceInfo]: + def unpack_sinfo_table(self) -> dict[PartPosition, TagSourceInfo]: return {sinfo.starttag_pos: sinfo for sinfo in self.sinfos} From b28e4db6c0aff7e7347369f1f416cfb081386903 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Fri, 3 Jul 2026 14:58:39 -0700 Subject: [PATCH 38/78] Try to reduce position translation complexity. --- tdom/parser.py | 19 +++- tdom/parser_utils.py | 202 +++++++++++++++++++++----------------- tdom/parser_utils_test.py | 51 ++++++++++ 3 files changed, 179 insertions(+), 93 deletions(-) create mode 100644 tdom/parser_utils_test.py diff --git a/tdom/parser.py b/tdom/parser.py index 90de8472..23b90acf 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -4,7 +4,11 @@ from string.templatelib import Interpolation, Template from .htmlspec import VOID_ELEMENTS -from .parser_utils import HTMLAttribute, parser_pos_to_part_pos +from .parser_utils import ( + HTMLAttribute, + ParserPositionTranslator, + make_parser_pos_translator, +) from .placeholders import ( PlaceholderState, ) @@ -125,6 +129,13 @@ class SourceTracker: i_index: int = -1 # The current interpolation index. s_index: int = -1 # The current string index. + parser_pos_translator: ParserPositionTranslator = field(init=False) + + def __post_init__(self): + self.parser_pos_translator = make_parser_pos_translator( + self.template, self.placeholders.config + ) + def __iter__(self): return self @@ -193,16 +204,14 @@ def translate_pos(self, parser_pos: LinePosition) -> PartPosition: """ Translate the parser position into a part position in the template. """ - return parser_pos_to_part_pos( - self.template, self.placeholders.config, parser_pos - ) + return self.parser_pos_translator.translate(parser_pos) class TemplateParser(HTMLParser): root: OpenTFragment stack: list[OpenTag] source: SourceTracker | None - " Map from completed tnodes back to their opentag for error reporting. " + " Map from completed tnodes to their parsed children for error reporting. " tcomponent_children: dict[TComponent, list[TNode]] "List of children for each finished tcomponent, stored at closing. " sinfo_table: dict[PartPosition, TagSourceInfo] diff --git a/tdom/parser_utils.py b/tdom/parser_utils.py index f2c0394a..048b5574 100644 --- a/tdom/parser_utils.py +++ b/tdom/parser_utils.py @@ -1,3 +1,4 @@ +from dataclasses import dataclass from string.templatelib import Template from .placeholders import PlaceholderConfig @@ -7,102 +8,127 @@ type HTMLAttribute = tuple[str, str | None] +def make_parser_pos_translator( + template: Template, config: PlaceholderConfig +) -> ParserPositionTranslator: + # Precompute these. + source_text_parts = tuple( + template.strings[index // 2] + if index % 2 == 0 + else config.make_placeholder((index - 1) // 2) + for index in range(2 * len(template.strings) - 1) + ) + source_text_lines = tuple("".join(source_text_parts).split("\n")) + return ParserPositionTranslator( + template, config, source_text_parts, source_text_lines + ) + + +@dataclass +class ParserPositionTranslator: + template: Template + config: PlaceholderConfig + + source_text_parts: tuple[str, ...] + " The source text of each template part, with placeholders. " + + source_text_lines: tuple[str, ...] + " The source text of the entire template, with placeholders. " + + def validate(self, parser_pos: LinePosition): + """ + Check parser position targets existing line and offset in template. + + This attempts to reduce the complexity of the translating by letting us + assume the translation is possible. + """ + if parser_pos.line > len(self.source_text_lines): + raise ValueError("Line does not exist in source.") + elif parser_pos.line <= 0: + raise ValueError("Unreachable line number, must be > 0.") + # @NOTE: This includes an offset that is at the end of the line. + last_index = len(self.source_text_lines[parser_pos.line - 1]) + if parser_pos.offset > last_index: + raise ValueError( + f"Offset exceeds reachable characters or EOL in source line {parser_pos.line}: {parser_pos.offset} > {last_index}" + ) + elif parser_pos.offset < 0: + raise ValueError("Unreachable offset, must be >= 0.") + + def translate(self, parser_pos: LinePosition) -> PartPosition: + self.validate(parser_pos) + return self._translate(parser_pos) + + def _translate(self, parser_pos: LinePosition) -> PartPosition: + return parser_pos_to_part_pos(self.source_text_parts, parser_pos) + + def parser_pos_to_part_pos( - template: Template, - placeholder_config: PlaceholderConfig, + parts: tuple[str, ...], parser_pos: LinePosition, ) -> PartPosition: """ Translate the given parser position into a template part position. + + - Iterate over the template parts. + - Track the current line and offset while advancing into each part. + - When we reach the parser position then return the current part + and the current offset from the start of that part. + """ pos = MutableLinePosition() - combined_size = 2 * len(template.strings) - 1 - last_index = combined_size - 1 - for index in range(combined_size): - if index % 2 == 0: - s = template.strings[index // 2] - if parser_pos.line > pos.line: - # need more lines - nls_found = s.count("\n") # how many were found? - nls_need = parser_pos.line - pos.line # how many are needed? - if nls_found >= nls_need: - pos.line += nls_need - offset_found = len(s.split("\n", nls_need + 1)[nls_need]) - if offset_found >= parser_pos.offset: - # needed lines, found lines, found offset - pos.offset = parser_pos.offset - total_offset = ( - sum( - len(line) + 1 - for line in s.split("\n", nls_need + 1)[:nls_need] - ) - + parser_pos.offset - ) - return PartPosition(index, total_offset) - else: - # got enough lines, still need more offset - pos.offset = offset_found - elif nls_found > 0: - # some lines but still need more lines - pos.line += nls_found - pos.offset = len(s[s.rfind("\n") + 1 :]) - else: - # no lines, still need more lines - offset_found = len(s) - pos.offset += offset_found - elif parser_pos.line == pos.line: - # got enough lines, we just need more offset - offset_found = len(s[: s.find("\n")]) if "\n" in s else len(s) - offset_need = parser_pos.offset - pos.offset - if offset_found > offset_need: - pos.offset += offset_need - total_offset = offset_need # only from the start of this string. - # had lines, found offset + last_index = len(parts) - 1 + for index, part_text in enumerate(parts): + nls_found = part_text.count("\n") + if parser_pos.line > pos.line: # need more lines + nls_need = parser_pos.line - pos.line # how many are needed? + if nls_found >= nls_need: + pos.line += nls_need + lines_found = part_text.split("\n") + offset_found = len(lines_found[nls_need]) + if offset_found >= parser_pos.offset: + # needed lines, found lines, found offset + pos.offset = parser_pos.offset + total_offset = ( + sum(len(line) + 1 for line in lines_found[:nls_need]) + + parser_pos.offset + ) return PartPosition(index, total_offset) - elif offset_found == offset_need: - if index < last_index: - # @TODO: Start at the interpolation ? - return PartPosition(index + 1, 0) - else: - # @TEST - # @TODO: Is this possible? Seems like this position would - # technically be undefined and an error. - # Start at the very end of the last part (can this exist?) - return PartPosition(index, len(s)) else: - pos.offset += offset_found + # got enough lines, still need more offset + pos.offset = offset_found + elif nls_found > 0: + # some lines but still need more lines + last_nl_index = part_text.rfind("\n") + pos.line += nls_found + pos.offset = len(part_text[last_nl_index + 1 :]) else: - # We should have dropped out and failed earlier this would be a bug. - raise AssertionError( - f"Unexpected line: {pos.line} greater than asked for {parser_pos.line}" - ) - + # no lines, still need more lines + pos.offset += len(part_text) + elif parser_pos.line == pos.line: + # got enough lines, we just need more offset + first_nl_index = part_text.find("\n") + offset_found = ( + len(part_text[:first_nl_index]) if nls_found else len(part_text) + ) + offset_need = parser_pos.offset - pos.offset + if offset_found > offset_need: + pos.offset += offset_need + total_offset = offset_need + # had lines, found offset + return PartPosition(index, total_offset) + elif offset_found == offset_need: + if index != last_index: + return PartPosition(index + 1, 0) + else: + return PartPosition(last_index, offset_found) + else: + pos.offset += offset_found else: - i_index = (index - 1) // 2 - ph_length = len(placeholder_config.make_placeholder(i_index)) - if ( - pos.line == parser_pos.line - and pos.offset + ph_length > parser_pos.offset - ): - # Ie. we don't know how to determine how much of the - # interpolation expression would be equivalent to - # a substring of a placeholder. - raise ValueError( - f"Cannot split a placeholder for interpolations[{i_index}], placeholders are atomic." - ) - pos.offset += ph_length - if pos == parser_pos: - # An offset to the end of this interpolation should be the start - # of the following string. - # @TEST - # @TODO: Do we need this check or would it be picked up - # in the next iteration? - return PartPosition(index + 1, 0) - if pos == parser_pos: - # @TEST - # @TODO: When can this fall through happen? Or is this always an error? - return PartPosition(last_index, len(template.strings[-1])) - else: - raise ValueError( - "Unexpected position {pos}, did not reach required position {parser_pos}" - ) + # We should have dropped out and failed earlier this would be a bug. + raise AssertionError( + f"Unexpected line: {pos.line} greater than asked for {parser_pos.line}" + ) + raise AssertionError( + "Unexpected position {pos}, did not reach required position {parser_pos}" + ) diff --git a/tdom/parser_utils_test.py b/tdom/parser_utils_test.py new file mode 100644 index 00000000..d99c54d3 --- /dev/null +++ b/tdom/parser_utils_test.py @@ -0,0 +1,51 @@ +from collections.abc import Callable +from string.templatelib import Template + +import pytest + +from .parser_utils import ParserPositionTranslator, make_parser_pos_translator +from .placeholders import make_placeholder_config +from .source import LinePosition +from .template_utils import PartPosition + + +@pytest.fixture(scope="module") +def ph_config(): + return make_placeholder_config() + + +@pytest.fixture(scope="module") +def t_maker(ph_config) -> Callable[[Template], ParserPositionTranslator]: + def maker(template: Template) -> ParserPositionTranslator: + return make_parser_pos_translator(template=template, config=ph_config) + + return maker + + +class TestParserPosToPartPos: + def test_offset(self, t_maker): + ppt = t_maker(t"a*") + pos = ppt.translate(LinePosition(line=1, offset=1)) + assert pos.index == 0 and pos.offset == 1 + + def test_line(self, t_maker): + pos = t_maker(t"ab\n*").translate(LinePosition(line=2, offset=0)) + assert pos.index == 0 and pos.offset == 3 + + def test_interpolation_after(self, t_maker): + translator = t_maker(t"ab\nc{0}d*") + offset = len("".join(("c", translator.config.make_placeholder(0), "d"))) + pos = translator.translate(LinePosition(line=2, offset=offset)) + assert pos == PartPosition(index=2, offset=1) + + def test_interpolation_right_after(self, t_maker): + translator = t_maker(t"ab\nc{0}*") + offset = len("".join(("c", translator.config.make_placeholder(0)))) + pos = translator.translate(LinePosition(line=2, offset=offset)) + assert pos == PartPosition(index=2, offset=0) + + def test_interpolation_end_of_line_start_of_line(self, t_maker): + translator = t_maker(t"""ab\nc{0}a\n +""") + pos = translator.translate(LinePosition(line=3, offset=0)) + assert pos == PartPosition(index=2, offset=2) From 90de4930028fc654312931a257c8d692f3dea844 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Fri, 3 Jul 2026 23:18:58 -0700 Subject: [PATCH 39/78] Add ambiguous slash checks for normal elements as well. --- tdom/parser.py | 242 +++++++++++++++++++++++--------------------- tdom/parser_test.py | 96 ++++++++++++------ tdom/source.py | 3 + 3 files changed, 196 insertions(+), 145 deletions(-) diff --git a/tdom/parser.py b/tdom/parser.py index 23b90acf..312329fd 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -1,7 +1,7 @@ from collections.abc import Sequence from dataclasses import dataclass, field from html.parser import HTMLParser -from string.templatelib import Interpolation, Template +from string.templatelib import Template from .htmlspec import VOID_ELEMENTS from .parser_utils import ( @@ -154,32 +154,12 @@ def __next__(self): else: raise AssertionError("{self.i_index=} should not exceed {self.s_index=}") - @property - def interpolations(self) -> tuple[Interpolation, ...]: - return self.template.interpolations - def values_match(self, i_index1: int, i_index2: int) -> bool: return ( - self.interpolations[i_index1].value == self.interpolations[i_index2].value + self.template.interpolations[i_index1].value + == self.template.interpolations[i_index2].value ) - def get_expression( - self, i_index: int, fallback_prefix: str = "interpolation" - ) -> str: - """ - Resolve an interpolation index to its original expression for error messages. - Falls back to a synthetic expression if the original is empty. - """ - ip = self.interpolations[i_index] - return ip.expression if ip.expression else f"{{{fallback_prefix}-{i_index}}}" - - def format_starttag(self, i_index: int) -> str: - """Format a component start tag for error messages.""" - return self.get_expression(i_index, fallback_prefix="component-starttag") - - def format_endtag(self, i_index: int) -> str: - return self.get_expression(i_index, fallback_prefix="component-endtag") - def get_reader(self) -> SourceReader: return SourceReader(template=self.template) @@ -478,6 +458,36 @@ def extract_component_children_ref( children_ref = TemplateRef(strings=("",), i_indexes=()) return children_ref + def make_mismatch_error( + self, + starttag_sinfo: OpenTagSourceInfo, + endtag_ref: TemplateRef, + endtag_pos: PartPosition, + ) -> ParsingError: + reader = self.get_source().get_reader() + starttag_repr = reader.ref_to_repr(starttag_sinfo.starttag_ref) + starttag_pos_msg = reader.make_template_pos_msg(starttag_sinfo.starttag_pos) + endtag_repr = reader.ref_to_repr(endtag_ref) + endtag_pos_msg = reader.make_template_pos_msg(endtag_pos) + e = ParsingError( + f"Mismatched closing tag at {endtag_pos_msg} for {starttag_repr} at {starttag_pos_msg}." + ) + if self.has_ambiguous_forward_slash(starttag_sinfo): + e.add_note( + f'Did you mean to quote the last attribute or put a space before "/>" for "{starttag_repr}" at {starttag_pos_msg}?' + ) + return e + + def make_invalid_endtag_error( + self, endtag_ref: TemplateRef, endtag_pos: PartPosition + ) -> ParsingError: + reader = self.get_source().get_reader() + endtag_repr = reader.ref_to_repr(endtag_ref) + endtag_pos_msg = reader.make_template_pos_msg(endtag_pos) + raise ParsingError( + f"Component end tags must have exactly one interpolation, {endtag_repr} at {endtag_pos_msg}." + ) + def validate_end_tag(self, tag: str, open_tag: OpenTag) -> int | None: """Validate that closing tag matches open tag. Return component end index if applicable.""" source = self.get_source() @@ -485,44 +495,22 @@ def validate_end_tag(self, tag: str, open_tag: OpenTag) -> int | None: match open_tag: case OpenTElement(): - if not tag_ref.is_literal: - # - # EXAMPLE 3: getting earlier source slice and line number AND - # current source slice and current line number - # - reader = source.get_reader() - starttag_ref = open_tag.sinfo.starttag_ref - starttag_repr = reader.ref_to_repr(starttag_ref) - endtag_repr = reader.ref_to_repr(tag_ref) - endtag_pos_msg = reader.make_template_pos_msg(self.get_source_pos()) - starttag_pos_msg = reader.make_template_pos_msg(open_tag.source_pos) - raise ParsingError( - f"Component closing tag at {endtag_pos_msg} found for element {starttag_repr} at {starttag_pos_msg}." - ) - if tag != open_tag.tag: - raise ParsingError( - f"Mismatched closing tag for element <{open_tag.tag}>." + if tag_ref.is_singleton or (tag_ref.is_literal and tag != open_tag.tag): + raise self.make_mismatch_error( + open_tag.sinfo, tag_ref, self.get_source_pos() ) + elif not tag_ref.is_singleton and not tag_ref.is_literal: + raise self.make_invalid_endtag_error(tag_ref, self.get_source_pos()) return None - case OpenTFragment(): raise ParsingAssertionError("We do not support anonymous fragments.") - - case OpenTComponent(start_i_index=start_i_index): + case OpenTComponent(): if tag_ref.is_literal: - starttag = source.format_starttag(start_i_index) - e = ParsingError( - f"Mismatched closing tag for component with tag {{{starttag}}}." - ) - if self.has_ambiguous_forward_slash(open_tag.sinfo): - e.add_note( - f'Did you mean to quote the last attribute or put a space before "/>" for "<{{{starttag}}} .../>"?' - ) - raise e - if not tag_ref.is_singleton: - raise ParsingError( - "Component end tags must have exactly one interpolation." + raise self.make_mismatch_error( + open_tag.sinfo, tag_ref, self.get_source_pos() ) + elif not tag_ref.is_singleton: + raise self.make_invalid_endtag_error(tag_ref, self.get_source_pos()) return tag_ref.i_indexes[0] def get_starttag_ref(self) -> TemplateRef: @@ -587,35 +575,25 @@ def handle_startendtag(self, tag: str, attrs: Sequence[HTMLAttribute]) -> None: self.append_child(final_tag) def handle_endtag(self, tag: str) -> None: + endtag_pos = self.get_source_pos() if not self.stack: source = self.get_source() - tag_ref = source.find_placeholders(tag) - if tag_ref.is_literal: - # EXAMPLE 1: getting line number, does not slice/extract source - reader = source.get_reader() - pos_msg = reader.make_template_pos_msg(self.get_source_pos()) - raise ParsingError( - f"Unexpected closing tag with no open tag, {pos_msg}." - ) - if not tag_ref.is_singleton: - # @TODO: Also it doesn't match anything + reader = source.get_reader() + endtag_ref = source.find_placeholders(tag) + endtag_repr = reader.ref_to_repr(endtag_ref) + endtag_pos_msg = reader.make_template_pos_msg(endtag_pos) + if endtag_ref.is_literal or endtag_ref.is_singleton: raise ParsingError( - "Component end tags must have exactly one interpolation." + f"Unexpected closing tag with no open tag, {endtag_pos_msg}." ) - # Component tag endtag but no component tag is open... - # EXAMPLE 2: getting line number AND get source (repr)esentation via ref - reader = source.get_reader() - pos_msg = reader.make_template_pos_msg(self.get_source_pos()) - tag_repr = reader.ref_to_repr(tag_ref) - raise ParsingError( - f"Unexpected closing component tag with no open tag, {pos_msg}." - ) + else: + raise self.make_invalid_endtag_error(endtag_ref, endtag_pos) open_tag = self.stack.pop() endtag_i_index = self.validate_end_tag(tag, open_tag) final_tag = self.finalize_tag( open_tag, endtag_i_index=endtag_i_index, - endtag_pos=self.get_source_pos(), + endtag_pos=endtag_pos, ) self.append_child(final_tag) @@ -693,6 +671,79 @@ def reset(self): self.sinfo_table = {} self.tcomponent_children = {} + def run_unclosed_ambiguous_slash_checks( + self, parent: OpenTag, e: ParsingError + ) -> None: + """ + Check for cases where ambiguous slash might create a confusing error. + + @NOTE: This add exception notes to the exception but does not throw it. + """ + source = self.get_source() + reader = source.get_reader() + if isinstance( + parent, (OpenTElement, OpenTComponent) + ) and self.has_ambiguous_forward_slash(parent.sinfo): + # CASE: "<{C1} attr={value}/>" -- maybe user meant to self-close? + # CASE: "
" -- mayber user meant to self-close? + starttag_ref = parent.sinfo.starttag_ref + starttag_repr = reader.ref_to_repr(starttag_ref) + pos_msg = reader.make_template_pos_msg(parent.source_pos) + e.add_note( + f'Did you mean to quote the last attribute or put a space before "/>" for "{starttag_repr}" at {pos_msg}?' + ) + elif isinstance(parent, OpenTElement): + # ie. t"
", looks + # like we missed a closing
but really we meant to + # self-close the middle div. + children = parent.children[:] + while children: + child = children.pop(0) + if isinstance(child, TElement) and child.tag == parent.tag: + sinfo = ( + self.sinfo_table.get(child.source_pos) + if child.source_pos is not None + else None + ) + if sinfo and self.has_ambiguous_forward_slash(sinfo): + full_starttag_repr = reader.ref_to_repr(sinfo.starttag_ref) + e.add_note( + f'Did you mean to quote the last attribute or put a space before "/>" for "{full_starttag_repr}"?' + ) + children.extend(child.children) + elif isinstance(parent, OpenTComponent): + # This is a special case where a component accidentally closes + # another component but we don't check the actual values in + # the parser so we can't tell until we are generating an error + # (when we can check the values). + # + # CASE: t"<{C2}><{C1} attr=/>" + # Maybe user meant to self-close <{C1} ...>, but closed by leaving <{C2}...> open? + # CASE: t"<{C3}><{C2}><{C1} attr=/>" + for comp in reversed( + self.get_closed_tcomps(parent, recurse_component_children=True) + ): + if ( + comp.end_i_index is not None + and comp.start_i_index != comp.end_i_index + and not source.values_match(comp.start_i_index, comp.end_i_index) + ): + starttag_repr = reader.make_interpolation_repr(comp.start_i_index) + endtag_repr = reader.make_interpolation_repr(comp.end_i_index) + e.add_note( + f"Component start tag, <{starttag_repr} ...>, and end tag, , have values that do not match." + ) + sinfo = ( + self.sinfo_table.get(comp.source_pos) + if comp.source_pos is not None + else None + ) + if sinfo and self.has_ambiguous_forward_slash(sinfo): + full_starttag_repr = reader.ref_to_repr(sinfo.starttag_ref) + e.add_note( + f'Did you mean to quote the last attribute or put a space before "/>" for "{full_starttag_repr}"?' + ) + def close(self) -> None: source = self.get_source() if self.waiting_for_data(): @@ -707,46 +758,7 @@ def close(self) -> None: ) if self.stack: e = ParsingError("Invalid HTML structure: unclosed tags remain.") - # @TODO: We need to determine which tags this might apply to, - # this only applies to components. - parent = self.stack[-1] - if isinstance(parent, OpenTComponent) and self.has_ambiguous_forward_slash( - parent.sinfo - ): - # CASE: "<{C1} attr={value}/>" -- meant to self-close - # Maybe user meant to self-close? - starttag = source.format_starttag(parent.start_i_index) - e.add_note( - f'Did you mean to quote the last attribute or put a space before "/>" for "<{{{starttag}}} .../>"?' - ) - else: - # CASE: t"<{C2}><{C1} attr=/>" - # Maybe user meant to self-close <{C1} ...>, but closed by leaving <{C2}...> open? - # CASE: t"<{C3}><{C2}><{C1} attr=/>" - for comp in reversed( - self.get_closed_tcomps(parent, recurse_component_children=True) - ): - if ( - comp.end_i_index is not None - and comp.start_i_index != comp.end_i_index - and not source.values_match( - comp.start_i_index, comp.end_i_index - ) - ): - sinfo = ( - self.sinfo_table.get(comp.source_pos) - if comp.source_pos is not None - else None - ) - starttag = source.format_starttag(comp.start_i_index) - endtag = source.format_endtag(comp.end_i_index) - e.add_note( - f"Component start tag, <{{{starttag}}}>, and end tag, , have values that do not match." - ) - if self.has_ambiguous_forward_slash(sinfo): - e.add_note( - f'Did you mean to quote the last attribute or put a space before "/>" for "<{{{starttag}}} .../>"?' - ) + self.run_unclosed_ambiguous_slash_checks(self.stack[-1], e) raise e if not source.placeholders.is_empty: raise ParsingError("Some placeholders were never resolved.") diff --git a/tdom/parser_test.py b/tdom/parser_test.py index 27c1945f..0a57e99f 100644 --- a/tdom/parser_test.py +++ b/tdom/parser_test.py @@ -455,7 +455,7 @@ def test_component_element_invalid_closing_tag(): def Component(): pass - with pytest.raises(ParsingError, match="Mismatched closing tag"): + with pytest.raises(ParsingError, match="Mismatched closing tag
"): _ = TemplateParser.parse(t"<{Component}>
") @@ -463,9 +463,8 @@ def test_component_element_invalid_opening_tag(): def Component(): pass - with pytest.raises( - ParsingError, match="Component closing tag .* found for element" - ): + # @NOTE: intentional expression + with pytest.raises(ParsingError, match="Mismatched closing tag "): _ = TemplateParser.parse(t"
") @@ -489,7 +488,7 @@ def test_unmatched_end_component_tag_error(): def Component(): pass - with pytest.raises(ParsingError, match="Unexpected closing component tag"): + with pytest.raises(ParsingError, match="Unexpected closing tag "): _ = TemplateParser.parse(t"") @@ -629,25 +628,52 @@ def test_extract_with_templated_attr_gt_char(self, Component): ) -class TestComponentUnquotedAttrValueWithAmbiguousSlash: - @pytest.fixture - def comp_maker(self): - def maker(suffix=None): - def _Comp(children: Template, title: str) -> Template: - return children +class TestElementWithAmbiguousSlash: + def test_root_unclosed_error(self): + with pytest.raises( + ParsingError, match="Did you mean to quote the last attribute.*attr[=]root/" + ): + _ = TemplateParser.parse(t"
") + + def test_nested_unclosed_error(self): + with pytest.raises( + ParsingError, + match="Did you mean to quote the last attribute.*attr[=]nested/", + ): + _ = TemplateParser.parse(t"
") + + def test_double_nested_unclosed_error(self): + with pytest.raises( + ParsingError, + match="Did you mean to quote the last attribute.*attr[=]nested/", + ): + _ = TemplateParser.parse(t"
") - if suffix is not None: - _Comp.__name__ = f"{_Comp.__name__}__{suffix}" - return _Comp + def test_mismatch_with_element_error(self): + with pytest.raises( + ParsingError, + match="Did you mean to quote the last attribute.*attr[=]mismatch/", + ): + _ = TemplateParser.parse(t"
") + + def test_mismatch_with_component_error(self): + def Comp(children: Template) -> Template: + return t"" - return maker + with pytest.raises( + ParsingError, + match="Did you mean to quote the last attribute.*attr[=]mismatch/", + ): + _ = TemplateParser.parse(t"<{Comp}>
") + +class TestComponentWithAmbiguousSlash: @pytest.fixture - def Comp(self): - def _Comp(children: Template, title: str) -> Template: + def Comp1(self): + def _Comp1(children: Template, title: str) -> Template: return children - return _Comp + return _Comp1 @pytest.fixture def Comp2(self): @@ -656,28 +682,38 @@ def _Comp2(children: Template, title: str) -> Template: return _Comp2 - def test_comp_unquoted_attr_value_error_root(self, Comp): + @pytest.fixture + def Comp3(self): + def _Comp3(children: Template, title: str) -> Template: + return children + + return _Comp3 + + def test_mismatch_with_element_error(self, Comp1): with pytest.raises( - ParsingError, match="Did you mean to quote the last attribute" + ParsingError, + match="Did you mean to quote the last attribute.*title[=]today/", ): - _ = TemplateParser.parse(t"<{Comp} title=today/>") + _ = TemplateParser.parse(t"
<{Comp1} title=today/>
") - def test_comp_unquoted_attr_value_error_nested_in_el(self, Comp): + def test_root_unclosed_error(self, Comp1): with pytest.raises( - ParsingError, match="Did you mean to quote the last attribute" + ParsingError, + match="Did you mean to quote the last attribute.*title[=]today/", ): - _ = TemplateParser.parse(t"
<{Comp} title=today/>
") + _ = TemplateParser.parse(t"<{Comp1} title=today/>") - def test_comp_unquoted_attr_value_error_single_nested_in_comp(self, Comp, Comp2): + def test_single_nested_unclosed_error(self, Comp1, Comp2): with pytest.raises( - ParsingError, match="Did you mean to quote the last attribute" + ParsingError, + match="Did you mean to quote the last attribute.*title[=]today/", ): - _ = TemplateParser.parse(t"<{Comp2}><{Comp} title=today/>") + _ = TemplateParser.parse(t"<{Comp2}><{Comp1} title=today/>") - def test_comp_unquoted_attr_value_error_double_nested_in_comp(self, comp_maker): - Comp1, Comp2, Comp3 = comp_maker("1"), comp_maker("2"), comp_maker("3") + def test_double_nested_unclosed_error(self, Comp1, Comp2, Comp3): with pytest.raises( - ParsingError, match="Did you mean to quote the last attribute" + ParsingError, + match="Did you mean to quote the last attribute.*title[=]today/", ): _ = TemplateParser.parse( t"<{Comp2}><{Comp1}><{Comp3} title=today/>" diff --git a/tdom/source.py b/tdom/source.py index 79840b24..d93a18be 100644 --- a/tdom/source.py +++ b/tdom/source.py @@ -80,6 +80,9 @@ def make_template_pos_msg(self, source_pos: PartPosition) -> str: template_pos = self.to_template_pos(source_pos) return f"line {template_pos.line} offset {template_pos.offset}" + def make_interpolation_repr(self, i_index: int) -> str: + return interpolation_repr(self.template.interpolations[i_index]) + def to_template_pos(self, source_pos: PartPosition) -> LinePosition: """ Convert a (template) part position into a line position based on the From 2b348e07faa8c336b782dec3325143f68e8ee20b Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Fri, 3 Jul 2026 23:33:59 -0700 Subject: [PATCH 40/78] Simplify --- tdom/parser_utils.py | 10 +----- tdom/parser_utils_test.py | 73 +++++++++++++++++++++------------------ 2 files changed, 40 insertions(+), 43 deletions(-) diff --git a/tdom/parser_utils.py b/tdom/parser_utils.py index 048b5574..a464a48f 100644 --- a/tdom/parser_utils.py +++ b/tdom/parser_utils.py @@ -19,16 +19,11 @@ def make_parser_pos_translator( for index in range(2 * len(template.strings) - 1) ) source_text_lines = tuple("".join(source_text_parts).split("\n")) - return ParserPositionTranslator( - template, config, source_text_parts, source_text_lines - ) + return ParserPositionTranslator(source_text_parts, source_text_lines) @dataclass class ParserPositionTranslator: - template: Template - config: PlaceholderConfig - source_text_parts: tuple[str, ...] " The source text of each template part, with placeholders. " @@ -57,9 +52,6 @@ def validate(self, parser_pos: LinePosition): def translate(self, parser_pos: LinePosition) -> PartPosition: self.validate(parser_pos) - return self._translate(parser_pos) - - def _translate(self, parser_pos: LinePosition) -> PartPosition: return parser_pos_to_part_pos(self.source_text_parts, parser_pos) diff --git a/tdom/parser_utils_test.py b/tdom/parser_utils_test.py index d99c54d3..fc08fbab 100644 --- a/tdom/parser_utils_test.py +++ b/tdom/parser_utils_test.py @@ -1,10 +1,9 @@ -from collections.abc import Callable from string.templatelib import Template import pytest from .parser_utils import ParserPositionTranslator, make_parser_pos_translator -from .placeholders import make_placeholder_config +from .placeholders import PlaceholderConfig, make_placeholder_config from .source import LinePosition from .template_utils import PartPosition @@ -14,38 +13,44 @@ def ph_config(): return make_placeholder_config() -@pytest.fixture(scope="module") -def t_maker(ph_config) -> Callable[[Template], ParserPositionTranslator]: - def maker(template: Template) -> ParserPositionTranslator: - return make_parser_pos_translator(template=template, config=ph_config) - - return maker +def make_ppt(template: Template, config: PlaceholderConfig) -> ParserPositionTranslator: + "Just a shorthand function." + return make_parser_pos_translator(template=template, config=config) class TestParserPosToPartPos: - def test_offset(self, t_maker): - ppt = t_maker(t"a*") - pos = ppt.translate(LinePosition(line=1, offset=1)) - assert pos.index == 0 and pos.offset == 1 - - def test_line(self, t_maker): - pos = t_maker(t"ab\n*").translate(LinePosition(line=2, offset=0)) - assert pos.index == 0 and pos.offset == 3 - - def test_interpolation_after(self, t_maker): - translator = t_maker(t"ab\nc{0}d*") - offset = len("".join(("c", translator.config.make_placeholder(0), "d"))) - pos = translator.translate(LinePosition(line=2, offset=offset)) - assert pos == PartPosition(index=2, offset=1) - - def test_interpolation_right_after(self, t_maker): - translator = t_maker(t"ab\nc{0}*") - offset = len("".join(("c", translator.config.make_placeholder(0)))) - pos = translator.translate(LinePosition(line=2, offset=offset)) - assert pos == PartPosition(index=2, offset=0) - - def test_interpolation_end_of_line_start_of_line(self, t_maker): - translator = t_maker(t"""ab\nc{0}a\n -""") - pos = translator.translate(LinePosition(line=3, offset=0)) - assert pos == PartPosition(index=2, offset=2) + def test_offset(self, ph_config): + ppt = make_ppt(t"a*", ph_config) + assert ppt.translate(LinePosition(line=1, offset=1)) == PartPosition( + index=0, offset=1 + ) + + def test_line(self, ph_config): + ppt = make_ppt(t"ab\n*", ph_config) + assert ppt.translate(LinePosition(line=2, offset=0)) == PartPosition( + index=0, offset=3 + ) + + def test_interpolation_after(self, ph_config): + ppt = make_ppt(t"ab\nc{0}d*", ph_config) + offset = len("".join(("c", ph_config.make_placeholder(0), "d"))) + assert ppt.translate(LinePosition(line=2, offset=offset)) == PartPosition( + index=2, offset=1 + ) + + def test_interpolation_right_after(self, ph_config): + ppt = make_ppt(t"ab\nc{0}*", ph_config) + offset = len("".join(("c", ph_config.make_placeholder(0)))) + assert ppt.translate(LinePosition(line=2, offset=offset)) == PartPosition( + index=2, offset=0 + ) + + def test_interpolation_end_of_line_start_of_line(self, ph_config): + ppt = make_ppt( + t"""ab\nc{0}a\n +""", + ph_config, + ) + assert ppt.translate(LinePosition(line=3, offset=0)) == PartPosition( + index=2, offset=2 + ) From 0a3243569c4bdd600400e7f4eacfe6a756beb928 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Fri, 3 Jul 2026 23:44:35 -0700 Subject: [PATCH 41/78] Drop unused method. --- tdom/parser_test.py | 2 +- tdom/template_utils.py | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/tdom/parser_test.py b/tdom/parser_test.py index 0a57e99f..cb9475b6 100644 --- a/tdom/parser_test.py +++ b/tdom/parser_test.py @@ -740,4 +740,4 @@ def test_tnode_parser_position(): ) el = tnode.children[0] assert isinstance(el, tnode_type) - assert el.source_pos == PartPosition.pack_s_index(0, offset=len("
")) + assert el.source_pos == PartPosition(index=0, offset=len("
")) diff --git a/tdom/template_utils.py b/tdom/template_utils.py index ad3ea20d..c2e7cacc 100644 --- a/tdom/template_utils.py +++ b/tdom/template_utils.py @@ -152,11 +152,3 @@ class PartPosition: offset: int = 0 " Offset from the start of the template part. " - - @classmethod - def pack_s_index(cls, s_index: int, offset: int = 0): - return cls(index=s_index * 2, offset=offset) - - @classmethod - def pack_i_index(cls, i_index: int, offset: int = 0): - return cls(index=i_index * 2 + 1, offset=offset) From e915387f9f5d493a323f02805fe4f14b1fa46f18 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Sat, 4 Jul 2026 12:09:00 -0700 Subject: [PATCH 42/78] Use chain.from_iterable instead of sum. --- tdom/template_utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tdom/template_utils.py b/tdom/template_utils.py index c2e7cacc..0fb64042 100644 --- a/tdom/template_utils.py +++ b/tdom/template_utils.py @@ -1,6 +1,7 @@ import typing as t from collections.abc import Sequence from dataclasses import dataclass +from itertools import chain from string.templatelib import Interpolation, Template @@ -16,9 +17,11 @@ def template_from_parts( def combine_template_refs(*template_refs: TemplateRef) -> TemplateRef: + """ Concatenate multiple template refs together into a single ref. """ + # trefs -> naive templates -> naive template -> tref return TemplateRef.from_naive_template( - sum((tr.to_naive_template() for tr in template_refs), t"") - ) + Template(*chain.from_iterable( + tr.to_naive_template() for tr in template_refs))) @dataclass(slots=True, frozen=True) From 715c35995d9a51c1c4a648c4997895a13e1bb6c2 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Sat, 4 Jul 2026 14:55:48 -0700 Subject: [PATCH 43/78] Add note to docs. --- tdom/template_utils.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tdom/template_utils.py b/tdom/template_utils.py index 0fb64042..8713e0e8 100644 --- a/tdom/template_utils.py +++ b/tdom/template_utils.py @@ -17,11 +17,11 @@ def template_from_parts( def combine_template_refs(*template_refs: TemplateRef) -> TemplateRef: - """ Concatenate multiple template refs together into a single ref. """ + """Concatenate multiple template refs together into a single ref.""" # trefs -> naive templates -> naive template -> tref return TemplateRef.from_naive_template( - Template(*chain.from_iterable( - tr.to_naive_template() for tr in template_refs))) + Template(*chain.from_iterable(tr.to_naive_template() for tr in template_refs)) + ) @dataclass(slots=True, frozen=True) @@ -139,7 +139,7 @@ def slice_from_template( @dataclass(slots=True, frozen=True) class PartPosition: """ - A template part position. + A unified template part position. Translate indexes into strings by multiplying by 2. ie. 0->0, 1->2, 2->4, etc. @@ -148,6 +148,9 @@ class PartPosition: Translate indexes into interpolations by multiplying by 2 and then adding 1. ie. 0->1, 1->3, 2->5, etc. Reverse by subtracting 1 and dividing by 2. + + Using unified indexes allows for simpler iteration as well as starting + or stopping at either type of part more seamlessly. """ index: int From d49af2a3096f4de2239fc36c522b94cb2a2b7fb9 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Sat, 4 Jul 2026 14:56:32 -0700 Subject: [PATCH 44/78] Refactor error display and expand type handling. --- tdom/processor.py | 101 ++++++++++++++++++++++++++++++---------------- 1 file changed, 66 insertions(+), 35 deletions(-) diff --git a/tdom/processor.py b/tdom/processor.py index 0a2ae931..6bad72e0 100644 --- a/tdom/processor.py +++ b/tdom/processor.py @@ -741,6 +741,66 @@ class TemplateProcessor(ITemplateProcessor): uppercase_doctype: bool = False # DOCTYPE vs doctype + def _add_process_error_notes( + self, + e: ProcessingError, + ) -> None: + for e_state in reversed(e.template_e_states): + if not e_state.ttree: + # Just skip this special case where processing could not + # even get started because the template wouldn't parse. + continue + elif not (e_state.tnode and e_state.template): + raise AssertionError( + "This should not happen if we have properly contained the error." + ) + else: + self._add_tnode_error_note( + e, e_state.ttree, e_state.tnode, e_state.template + ) + + def _add_tnode_error_note( + self, + e: ProcessingError, + ttree: TTree, # The root metadata for the "current" template + tnode: TNode, # The leafmost tnode where the error was caught for the "current" template + template: Template, # The "current" template that was being processed + ) -> None: + sinfo_table = ttree.unpack_sinfo_table() + sinfo = None + source_pos = None + if isinstance( + tnode, + (TElement, TText, TComment, TComponent, TDocumentType), + ): + source_pos = tnode.source_pos + if source_pos: + sinfo = sinfo_table.get(source_pos, None) + reader = SourceReader(template) + if sinfo: + # + # Example 4: Getting starttag repr and pos in processor. + # + starttag_repr = reader.ref_to_repr(sinfo.starttag_ref) + starttag_pos_msg = reader.make_template_pos_msg(sinfo.starttag_pos) + else: + # @TODO: Scrape together what we can for a better message. + if isinstance(tnode, TElement): + starttag_repr = f"<{tnode.tag} ...>" + elif isinstance(tnode, TComponent): + starttag_repr = f"<{{Comp(i_index={tnode.start_i_index})}} ...>" + elif isinstance(tnode, (TText, TComment)): + starttag_repr = reader.ref_to_repr(tnode.ref) + elif isinstance(tnode, (TDocumentType)): + starttag_repr = f"" + else: + starttag_repr = tnode.__class__.__name__.upper() + if source_pos: + starttag_pos_msg = reader.make_template_pos_msg(source_pos) + else: + starttag_pos_msg = "unknown location" + e.add_note(f"Error occurred at {starttag_repr} at {starttag_pos_msg}.") + def process( self, root_template: Template, @@ -752,40 +812,7 @@ def process( try: return self._process_template(root_template, assume_ctx) except ProcessingError as e: - # - # @TODO: I think we could optionally consolidate and/or reformat - # all the exceptions here if needed and move this entire thing to - # a special error formatting tool. - # - for e_state in reversed(e.template_e_states): - if not e_state.ttree: - # Just skip this special case where processing could not - # even get started because the template wouldn't parse. - continue - sinfo_table = e_state.ttree.unpack_sinfo_table() - sinfo = source_pos = None - if isinstance( - e_state.tnode, - (TElement, TText, TComment, TComponent, TDocumentType), - ): - source_pos = e_state.tnode.source_pos - if source_pos: - sinfo = sinfo_table.get(source_pos, None) - if sinfo: - # - # Example 4: Getting starttag repr and pos in processor. - # - reader = SourceReader( - e_state.template, - ) - starttag_repr = reader.ref_to_repr(sinfo.starttag_ref) - starttag_pos_msg = reader.make_template_pos_msg(sinfo.starttag_pos) - e.add_note( - f"Error occurred at {starttag_repr} at {starttag_pos_msg}." - ) - else: - # @TODO: Scrape together what we can for a better message. - e.add_note(f"Error occurred at {type(e_state.tnode)} in template") + self._add_process_error_notes(e) raise def _process_template(self, template: Template, last_ctx: ProcessContext) -> str: @@ -803,7 +830,11 @@ def _process_template(self, template: Template, last_ctx: ProcessContext) -> str except ProcessingError as e: e.template_e_states.append( TemplateErrorState( - template, ttree, e.last_tnode, e.values_index, e.iter_index + template=template, + ttree=ttree, + tnode=e.last_tnode, + values_index=e.values_index, + iter_index=e.iter_index, ) ) # Reset everything. From a8dcb87024666f81eb09a355d3be02135fbc683e Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Sat, 4 Jul 2026 23:46:35 -0700 Subject: [PATCH 45/78] Add processing error handling tests. --- tdom/processor_test.py | 59 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/tdom/processor_test.py b/tdom/processor_test.py index 7f0bc759..11fd9154 100644 --- a/tdom/processor_test.py +++ b/tdom/processor_test.py @@ -12,6 +12,7 @@ from .callables import get_callable_info from .escaping import escape_html_text +from .parser import ParsingError from .processor import ( AttributeProcessingError, CachedTemplateParserProxy, @@ -27,6 +28,7 @@ _prep_component_kwargs as prep_component_kwargs, ) from .protocols import HasHTMLDunder +from .tnodes import TElement, TText processor_api = _make_default_template_processor( parser_api=TemplateParserProxy(), # do not use cache @@ -2222,3 +2224,60 @@ def test_mathml(): is not a decimal number.

""" ) + + +class BadHTMLDunder: + def __html__(self): + raise ValueError("bad value") + + +class TestProcessingException: + def test_attr_error_has_matching_tnode(self): + "AttriubteProcessingError should point to tnode where error first occurred." + invalid_t = t"
" # 0 is invalid aria value + with pytest.raises(AttributeProcessingError) as exc_info: + _ = html(invalid_t) + assert len(exc_info.value.template_e_states) == 1 + tnode = exc_info.value.template_e_states[0].tnode + assert tnode and isinstance(tnode, TElement) and tnode.tag == "div" + + def test_text_error_has_matching_tnode(self): + "TextProcessingError should point to tnode where error first occurred." + bad_html_dunder = BadHTMLDunder() # __html__ raises exception + invalid_t = t"
{bad_html_dunder}
" + with pytest.raises(TextProcessingError) as exc_info: + _ = html(invalid_t) + assert len(exc_info.value.template_e_states) == 1 + tnode = exc_info.value.template_e_states[0].tnode + assert tnode and isinstance(tnode, TText) + + def test_processing_error_multiple_templates(self): + "*ProcessingError should stack error state for each template/tnode as stack unwinds." + inner_t = t"
" # 0 is invalid aria value + wrapper_t = t"
{inner_t}
" + with pytest.raises(AttributeProcessingError) as exc_info: + _ = html(wrapper_t) + assert len(exc_info.value.template_e_states) == 2 + inner_tnode = exc_info.value.template_e_states[0].tnode + assert ( + inner_tnode + and isinstance(inner_tnode, TElement) + and inner_tnode.tag == "div" + ) + wrapper_tnode = exc_info.value.template_e_states[1].tnode + assert wrapper_tnode and isinstance(wrapper_tnode, TText) + + def test_parsing_error_while_processing(self): + inner_t = t"
" + wrapper_t = t"
{inner_t}
" + with pytest.raises(ProcessingError) as exc_info: + _ = html(wrapper_t) + assert len(exc_info.value.template_e_states) == 2 + assert not exc_info.value.template_e_states[0].ttree, ( + "This can't be set for a parsing error." + ) + wrapper_tnode = exc_info.value.template_e_states[1].tnode + assert wrapper_tnode and isinstance(wrapper_tnode, TText) + assert isinstance(exc_info.value.__cause__, ParsingError), ( + "ProcessingError should be chained to parsing error." + ) From edf0c097af0dbdc644d937c0aad5e9cdb9893938 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Sun, 5 Jul 2026 14:55:30 -0700 Subject: [PATCH 46/78] Cleanup logic a bit. --- tdom/processor.py | 61 ++++++++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/tdom/processor.py b/tdom/processor.py index 6bad72e0..d51dea9e 100644 --- a/tdom/processor.py +++ b/tdom/processor.py @@ -766,39 +766,46 @@ def _add_tnode_error_note( tnode: TNode, # The leafmost tnode where the error was caught for the "current" template template: Template, # The "current" template that was being processed ) -> None: - sinfo_table = ttree.unpack_sinfo_table() - sinfo = None - source_pos = None - if isinstance( - tnode, - (TElement, TText, TComment, TComponent, TDocumentType), - ): - source_pos = tnode.source_pos - if source_pos: - sinfo = sinfo_table.get(source_pos, None) reader = SourceReader(template) - if sinfo: - # - # Example 4: Getting starttag repr and pos in processor. - # - starttag_repr = reader.ref_to_repr(sinfo.starttag_ref) - starttag_pos_msg = reader.make_template_pos_msg(sinfo.starttag_pos) + source_pos = ( + tnode.source_pos + if isinstance( + tnode, (TElement, TComponent, TFragment, TComment, TDocumentType, TText) + ) + else None + ) + + if isinstance(tnode, (TElement, TComponent)): + sinfo_table = ttree.unpack_sinfo_table() + sinfo = sinfo_table.get(source_pos, None) if source_pos else None + if sinfo: + starttag_repr = reader.ref_to_repr(sinfo.starttag_ref) + starttag_pos_msg = reader.make_template_pos_msg(sinfo.starttag_pos) + else: + if isinstance(tnode, TComponent): + starttag_repr = reader.ref_to_repr( + TemplateRef( + strings=("<", "...>"), i_indexes=(tnode.start_i_index,) + ) + ) + elif isinstance(tnode, TElement): + starttag_repr = f"<{tnode.tag} ...>" + else: + starttag_repr = "unknown source" # This would likely be a bug. else: - # @TODO: Scrape together what we can for a better message. - if isinstance(tnode, TElement): - starttag_repr = f"<{tnode.tag} ...>" - elif isinstance(tnode, TComponent): - starttag_repr = f"<{{Comp(i_index={tnode.start_i_index})}} ...>" - elif isinstance(tnode, (TText, TComment)): + if isinstance(tnode, (TText, TComment)): starttag_repr = reader.ref_to_repr(tnode.ref) - elif isinstance(tnode, (TDocumentType)): + elif isinstance(tnode, TDocumentType): starttag_repr = f"" else: + # @TODO: TFragment/TNode/? starttag_repr = tnode.__class__.__name__.upper() - if source_pos: - starttag_pos_msg = reader.make_template_pos_msg(source_pos) - else: - starttag_pos_msg = "unknown location" + + if source_pos: + starttag_pos_msg = reader.make_template_pos_msg(source_pos) + else: + starttag_pos_msg = "unknown location" # source_pos is optional right now + e.add_note(f"Error occurred at {starttag_repr} at {starttag_pos_msg}.") def process( From d1a1e9c3bd9d808b984fdac777ad5bf92986ce36 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Sun, 5 Jul 2026 15:28:44 -0700 Subject: [PATCH 47/78] Bring parser pos translator into parser itself. --- tdom/parser.py | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/tdom/parser.py b/tdom/parser.py index 312329fd..d83a870d 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -129,13 +129,6 @@ class SourceTracker: i_index: int = -1 # The current interpolation index. s_index: int = -1 # The current string index. - parser_pos_translator: ParserPositionTranslator = field(init=False) - - def __post_init__(self): - self.parser_pos_translator = make_parser_pos_translator( - self.template, self.placeholders.config - ) - def __iter__(self): return self @@ -180,20 +173,23 @@ def find_placeholders(self, text: str) -> TemplateRef: """ return self.placeholders.config.find_placeholders(text) - def translate_pos(self, parser_pos: LinePosition) -> PartPosition: - """ - Translate the parser position into a part position in the template. - """ - return self.parser_pos_translator.translate(parser_pos) - class TemplateParser(HTMLParser): root: OpenTFragment + "Fallback container of parsed nodes if no other topmost container is found." + stack: list[OpenTag] + "Stack of tags left open during parsing." + source: SourceTracker | None - " Map from completed tnodes to their parsed children for error reporting. " + "Source iterator of template parts, injecting placeholders as needed." + + parser_pos_translator: ParserPositionTranslator | None + "Translator from parser position to template part position. " + tcomponent_children: dict[TComponent, list[TNode]] "List of children for each finished tcomponent, stored at closing. " + sinfo_table: dict[PartPosition, TagSourceInfo] " Tags with more source info than just a position are tracked in this mapping. " @@ -226,8 +222,8 @@ def get_parser_pos(self) -> LinePosition: return LinePosition(line=line, offset=offset) def get_source_pos(self, parser_pos: LinePosition | None = None) -> PartPosition: - source = self.get_source() - return source.translate_pos( + "Translate the parser position into a part position in the source template." + return self.get_parser_pos_translator().translate( self.get_parser_pos() if parser_pos is None else parser_pos ) @@ -668,6 +664,7 @@ def reset(self): self.root = OpenTFragment() self.stack = [] self.source = None + self.parser_pos_translator = None self.sinfo_table = {} self.tcomponent_children = {} @@ -804,10 +801,18 @@ def get_source(self) -> SourceTracker: raise AssertionError("Source has not been initialized.") return self.source + def get_parser_pos_translator(self) -> ParserPositionTranslator: + if self.parser_pos_translator is None: + raise AssertionError("Parser position translator has not been initialized.") + return self.parser_pos_translator + def feed_template(self, template: Template) -> None: """Feed a Template's content to the parser.""" assert self.source is None, "Did you forget to call reset?" self.source = SourceTracker(template) + self.parser_pos_translator = make_parser_pos_translator( + template, self.source.placeholders.config + ) for content in self.source: self.feed(content) From a6bf918b744a894caf2844d28cf166ed5901dce8 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Sun, 5 Jul 2026 15:35:34 -0700 Subject: [PATCH 48/78] Add fragment test. --- tdom/parser_test.py | 48 +++++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/tdom/parser_test.py b/tdom/parser_test.py index cb9475b6..fcfe30bf 100644 --- a/tdom/parser_test.py +++ b/tdom/parser_test.py @@ -720,24 +720,30 @@ def test_double_nested_unclosed_error(self, Comp1, Comp2, Comp3): ) -def PositionComp() -> Template: - return t"" - - -def test_tnode_parser_position(): - for tnode_type, fragment in ( - (TElement, t""), - (TComment, t""), - (TDocumentType, t""), - (TComponent, t"<{PositionComp}>"), - (TText, t"Just a simple text."), - ): - tnode = TemplateParser.parse(t"
" + fragment + t"
") - assert ( - isinstance(tnode, TElement) - and tnode.tag == "div" - and len(tnode.children) == 1 - ) - el = tnode.children[0] - assert isinstance(el, tnode_type) - assert el.source_pos == PartPosition(index=0, offset=len("
")) +class TestSourcePosition: + def test_tnode_source_position(self): + " Check that non-fragments are assigned a source position. " + def PositionComp() -> Template: + return t"" + for tnode_type, fragment in ( + (TElement, t""), + (TComment, t""), + (TDocumentType, t""), + (TComponent, t"<{PositionComp}>"), + (TText, t"Just a simple text."), + ): + tnode = TemplateParser.parse(t"
" + fragment + t"
") + assert ( + isinstance(tnode, TElement) + and tnode.tag == "div" + and len(tnode.children) == 1 + ) + el = tnode.children[0] + assert isinstance(el, tnode_type) + assert el.source_pos == PartPosition(index=0, offset=len("
")) + + def test_fragment_source_position(self): + " Fragments do not have a position right now. " + root = TemplateParser.parse(t"
") + assert isinstance(root, TFragment) + assert not root.source_pos From d6aea2a0b25d2e9c17770ff1e3e132a50a36a6d0 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Sun, 5 Jul 2026 15:36:02 -0700 Subject: [PATCH 49/78] Formatting. --- tdom/parser_test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tdom/parser_test.py b/tdom/parser_test.py index fcfe30bf..929e5c59 100644 --- a/tdom/parser_test.py +++ b/tdom/parser_test.py @@ -722,9 +722,11 @@ def test_double_nested_unclosed_error(self, Comp1, Comp2, Comp3): class TestSourcePosition: def test_tnode_source_position(self): - " Check that non-fragments are assigned a source position. " + "Check that non-fragments are assigned a source position." + def PositionComp() -> Template: return t"" + for tnode_type, fragment in ( (TElement, t""), (TComment, t""), @@ -743,7 +745,7 @@ def PositionComp() -> Template: assert el.source_pos == PartPosition(index=0, offset=len("
")) def test_fragment_source_position(self): - " Fragments do not have a position right now. " + "Fragments do not have a position right now." root = TemplateParser.parse(t"
") assert isinstance(root, TFragment) assert not root.source_pos From 8296c562cd86c2b0410a94e8b242c7929c397c8a Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Sun, 5 Jul 2026 15:59:58 -0700 Subject: [PATCH 50/78] Move values match to reader and cleanup iteration in tracker. --- tdom/parser.py | 44 +++++++++++++++++++++----------------------- tdom/source.py | 11 +++++++++++ 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/tdom/parser.py b/tdom/parser.py index d83a870d..b0d7db9e 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -124,34 +124,32 @@ class SourceTracker: placeholders: PlaceholderState = field(default_factory=lambda: PlaceholderState()) - # if i_index >= s_index, feeding an interpolation; - # otherwise, when i_index < s_index, feeding a string. - i_index: int = -1 # The current interpolation index. - s_index: int = -1 # The current string index. + index: int = -1 def __iter__(self): + # + # @NOTE: This iterator is only meant to be used once since we track + # placeholders both by adding them and letting the user remove them + # with calls to `remove_placeholders()`. return self def __next__(self): - if self.i_index < self.s_index: - # Advance into the next interpolation UNLESS the last string - # we returned was at the end of the template. - if self.s_index == len(self.template.strings) - 1: - raise StopIteration - self.i_index += 1 - return self.placeholders.add_placeholder(self.i_index) - elif self.i_index == self.s_index: - # Advance into the next string - self.s_index += 1 - return self.template.strings[self.s_index] + if self.index < 2 * len(self.template.strings) - 2: + self.index += 1 + if self.index % 2 == 0: + return self.template.strings[self.index // 2] + else: + return self.placeholders.add_placeholder((self.index - 1) // 2) else: - raise AssertionError("{self.i_index=} should not exceed {self.s_index=}") + raise StopIteration - def values_match(self, i_index1: int, i_index2: int) -> bool: - return ( - self.template.interpolations[i_index1].value - == self.template.interpolations[i_index2].value - ) + def get_strings_index(self) -> int: + if self.index % 2 == 0: + return self.index // 2 + else: + raise AssertionError( + f"Index {self.index} is not references an entry in strings." + ) def get_reader(self) -> SourceReader: return SourceReader(template=self.template) @@ -323,7 +321,7 @@ def make_open_tag( # i_index + 1 because attributes WITHIN the component's tag might # contain interpolations causing the i_index (and s_index) to advance # arbitrarily. - children_start_s_index = self.get_source().s_index + children_start_s_index = self.get_source().get_strings_index() # @NOTE: This must be called when the tag is handled since it is # populated based on the most recently finished start tag. Otherwise @@ -723,7 +721,7 @@ def run_unclosed_ambiguous_slash_checks( if ( comp.end_i_index is not None and comp.start_i_index != comp.end_i_index - and not source.values_match(comp.start_i_index, comp.end_i_index) + and not reader.values_match(comp.start_i_index, comp.end_i_index) ): starttag_repr = reader.make_interpolation_repr(comp.start_i_index) endtag_repr = reader.make_interpolation_repr(comp.end_i_index) diff --git a/tdom/source.py b/tdom/source.py index d93a18be..508572da 100644 --- a/tdom/source.py +++ b/tdom/source.py @@ -66,6 +66,17 @@ class SourceReader: template: Template + def values_match(self, i_index1: int, i_index2: int) -> bool: + """Check if the two interpolation values match. + + @NOTE: This is meant to be used for reporting *better* error messages + after an error has already occurred. + """ + return ( + self.template.interpolations[i_index1].value + == self.template.interpolations[i_index2].value + ) + def ref_to_repr(self, ref: TemplateRef, limit: int | None = None) -> str: """ Convert tref to string representation of the underlying template. From 3e9492db8c52ad996d7a6827fd6b82e455cc4b36 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Sun, 5 Jul 2026 23:32:21 -0700 Subject: [PATCH 51/78] Don't go to the next part if there was a NL. --- tdom/parser_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdom/parser_utils.py b/tdom/parser_utils.py index a464a48f..22598bd2 100644 --- a/tdom/parser_utils.py +++ b/tdom/parser_utils.py @@ -110,7 +110,7 @@ def parser_pos_to_part_pos( # had lines, found offset return PartPosition(index, total_offset) elif offset_found == offset_need: - if index != last_index: + if index != last_index and first_nl_index == -1: return PartPosition(index + 1, 0) else: return PartPosition(last_index, offset_found) From 2da2b75a5ceac564eef64dd929953525ed0c7235 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Sun, 5 Jul 2026 23:33:10 -0700 Subject: [PATCH 52/78] Include comment 'tags' in error message. --- tdom/processor.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tdom/processor.py b/tdom/processor.py index d51dea9e..f441e29d 100644 --- a/tdom/processor.py +++ b/tdom/processor.py @@ -793,8 +793,10 @@ def _add_tnode_error_note( else: starttag_repr = "unknown source" # This would likely be a bug. else: - if isinstance(tnode, (TText, TComment)): + if isinstance(tnode, TText): starttag_repr = reader.ref_to_repr(tnode.ref) + elif isinstance(tnode, TComment): + starttag_repr = ''.format(reader.ref_to_repr(tnode.ref)) elif isinstance(tnode, TDocumentType): starttag_repr = f"" else: From e7d02631732324e6ec531b3fe57ec69bcebb1dd4 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Sun, 5 Jul 2026 23:49:25 -0700 Subject: [PATCH 53/78] Move slice into TemplateRef and start proper tests. --- tdom/processor.py | 2 +- tdom/source.py | 6 +-- tdom/template_utils.py | 84 +++++++++++++++++++++++-------------- tdom/template_utils_test.py | 73 ++++++++++++++++++++++++++++++++ 4 files changed, 129 insertions(+), 36 deletions(-) diff --git a/tdom/processor.py b/tdom/processor.py index f441e29d..083a04e9 100644 --- a/tdom/processor.py +++ b/tdom/processor.py @@ -796,7 +796,7 @@ def _add_tnode_error_note( if isinstance(tnode, TText): starttag_repr = reader.ref_to_repr(tnode.ref) elif isinstance(tnode, TComment): - starttag_repr = ''.format(reader.ref_to_repr(tnode.ref)) + starttag_repr = f"" elif isinstance(tnode, TDocumentType): starttag_repr = f"" else: diff --git a/tdom/source.py b/tdom/source.py index 508572da..2a1d7a32 100644 --- a/tdom/source.py +++ b/tdom/source.py @@ -2,7 +2,7 @@ from dataclasses import dataclass from string.templatelib import Interpolation, Template -from .template_utils import PartPosition, TemplateRef, slice_from_template +from .template_utils import PartPosition, TemplateRef, slice_to_tref @dataclass(slots=True, frozen=True) @@ -100,11 +100,11 @@ def to_template_pos(self, source_pos: PartPosition) -> LinePosition: string representation of the template. """ pos = MutableLinePosition() - for part in slice_from_template(self.template, start=None, stop=source_pos): + for part in slice_to_tref(self.template, start=None, stop=source_pos): if isinstance(part, str): text = part else: - text = interpolation_repr(part) + text = interpolation_repr(self.template.interpolations[part]) nls = text.count("\n") if nls: pos.offset = len(text) - (text.rfind("\n") + 1) diff --git a/tdom/template_utils.py b/tdom/template_utils.py index 8713e0e8..ccbddece 100644 --- a/tdom/template_utils.py +++ b/tdom/template_utils.py @@ -95,45 +95,65 @@ def resolve(self, interpolations: tuple[Interpolation, ...]) -> Template: resolved = [interpolations[i_index] for i_index in self.i_indexes] return template_from_parts(self.strings, resolved) + def slice( + self, + start: PartPosition | None = None, + stop: PartPosition | None = None, + ) -> TemplateRef: + """ + Slice template ref based on the given start and stop. + """ + size = 2 * len(self.strings) - 1 + first = start.index if start and start.index is not None else 0 + assert 0 <= first < size + offset = start.offset if start else None + last = stop.index if stop and stop.index is not None else size - 1 + assert 0 <= last < size + limit = stop.offset if stop else None + + strings = [] + i_indexes = [] + if first == last: + if first % 2 == 0: + return TemplateRef( + strings=(self.strings[first][offset:limit],), i_indexes=() + ) + else: + # @NOTE: No offset OR limit applied to interpolations. + return TemplateRef(strings=("", ""), i_indexes=(first,)) + else: + if first % 2 == 0: + strings.append(self.strings[first // 2][offset:]) + else: + # @NOTE: No offset applied to interpolations. + strings.append("") + i_indexes.append((first - 1) // 2) + + for index in range(first + 1, last + 1): + if index % 2 == 0: + if index == last: + strings.append(self.strings[index // 2][:limit]) + else: + strings.append(self.strings[index // 2]) + else: + # @NOTE: No limit applied to interpolations. + if index != last: + i_indexes.append((index - 1) // 2) + return TemplateRef(strings=tuple(strings), i_indexes=tuple(i_indexes)) + -def slice_from_template( +def slice_to_tref( template: Template, start: PartPosition | None = None, stop: PartPosition | None = None, -) -> t.Generator[Interpolation | str]: +) -> TemplateRef: """ - Yield the template parts that make up the requested slice. + Slice a template ref from a template based on the given start and stop. """ - first = start.index if start and start.index is not None else 0 - offset = start.offset if start else None - last = ( - stop.index if stop and stop.index is not None else 2 * len(template.strings) - 1 + tref = TemplateRef( + strings=template.strings, i_indexes=tuple(range(len(template.strings) - 1)) ) - limit = stop.offset if stop else None - - if first == last: - if first % 2 == 0: - yield template.strings[first][offset:limit] - else: - # @NOTE: No offset OR limit applied to interpolations. - yield template.interpolations[first] - return - else: - if first % 2 == 0: - yield template.strings[first // 2][offset:] - else: - # @NOTE: No offset applied to interpolations. - yield template.interpolations[(first - 1) // 2] - - for index in range(first + 1, last + 1): - if index % 2 == 0: - if index == last: - yield template.strings[index // 2][:limit] - else: - yield template.strings[index // 2] - else: - # @NOTE: No limit applied to interpolations. - yield template.interpolations[(index - 1) // 2] + return tref.slice(start=start, stop=stop) @dataclass(slots=True, frozen=True) diff --git a/tdom/template_utils_test.py b/tdom/template_utils_test.py index afce3615..7d65d1fe 100644 --- a/tdom/template_utils_test.py +++ b/tdom/template_utils_test.py @@ -101,3 +101,76 @@ def test_template_ref_resolve(): resolved_t = src_ref.resolve(src_t.interpolations) assert resolved_t.values == ("a", "c", "e") assert resolved_t.strings == ("", "b", "d", "f") + + +from .template_utils import PartPosition, slice_to_tref + + +class TestSliceToTRef: + def test_string_only_stop(self): + parts = list( + TemplateRef.from_naive_template(t"
").slice( + start=None, stop=PartPosition(index=0, offset=5) + ) + ) + assert parts == ["
"] + + def test_string_only_start(self): + parts = list( + slice_to_tref(t"
", start=PartPosition(index=0, offset=5)) + ) + assert parts == ["
"] + + def test_string_only_start_stop(self): + parts = list( + slice_to_tref( + t"
", + start=PartPosition(index=0, offset=4), + stop=PartPosition(index=0, offset=6), + ) + ) + assert parts == ["><"] + + def test_single_interpolation_stop(self): + parts = TemplateRef.from_naive_template(t"
{0}
").slice( + start=None, stop=PartPosition(index=1, offset=0) + ) + assert list(parts) == ["
"] + + def test_single_interpolation_start(self): + parts = slice_to_tref( + t"
{0}
", start=None, stop=PartPosition(index=1, offset=0) + ) + assert list(parts) == ["
"] + + def test_end_after_interpolation(self): + parts = list( + slice_to_tref( + t"
{0}
", start=None, stop=PartPosition(index=2, offset=0) + ) + ) + assert parts == ["
", 0] + + def test_newlines(self): + parts = list( + slice_to_tref( + t"
\n{0}
", start=None, stop=PartPosition(index=0, offset=5) + ) + ) + assert parts == ["
"] + parts = list( + slice_to_tref( + t"
\n{0}
", start=None, stop=PartPosition(index=0, offset=6) + ) + ) + assert parts == ["
\n"] + parts = list( + slice_to_tref( + t"
\n{0}
", start=None, stop=PartPosition(index=0, offset=7) + ) + ) + assert parts == ["
\n"] + parts = list( + slice_to_tref(t"
\n{0}
", start=PartPosition(index=0, offset=7)) + ) + assert parts == [0, "
"] From cf2cddb067a277af5f98842b2f3dc92dc42796c0 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Mon, 6 Jul 2026 20:56:20 -0700 Subject: [PATCH 54/78] Add a validation check to make sure positions are logical. --- tdom/parser_utils.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tdom/parser_utils.py b/tdom/parser_utils.py index 22598bd2..123b561a 100644 --- a/tdom/parser_utils.py +++ b/tdom/parser_utils.py @@ -52,7 +52,15 @@ def validate(self, parser_pos: LinePosition): def translate(self, parser_pos: LinePosition) -> PartPosition: self.validate(parser_pos) - return parser_pos_to_part_pos(self.source_text_parts, parser_pos) + part_pos = parser_pos_to_part_pos(self.source_text_parts, parser_pos) + if part_pos.index % 2 != 0 and part_pos.offset != 0: + # You can only land on the start of an interpolation + # There is no way to translate a position within a placeholder + # to a position within the original interpolation representation. + raise ValueError( + "Invalid parser position results in offset within interpolation!" + ) + return part_pos def parser_pos_to_part_pos( From dbf31f566796b835d6971f63624612de06855a33 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Mon, 6 Jul 2026 21:28:02 -0700 Subject: [PATCH 55/78] Expand position translator coverage. --- tdom/parser_utils_test.py | 149 ++++++++++++++++++++++++++++++++------ 1 file changed, 126 insertions(+), 23 deletions(-) diff --git a/tdom/parser_utils_test.py b/tdom/parser_utils_test.py index fc08fbab..03f9befb 100644 --- a/tdom/parser_utils_test.py +++ b/tdom/parser_utils_test.py @@ -18,39 +18,142 @@ def make_ppt(template: Template, config: PlaceholderConfig) -> ParserPositionTra return make_parser_pos_translator(template=template, config=config) -class TestParserPosToPartPos: - def test_offset(self, ph_config): +class TestParserPositionTranslator: + def test_case_nontailing_string_ends_with_newline(self, ph_config): + ppt = make_ppt(t"a\n{0}b", ph_config) + assert ppt.translate(LinePosition(line=2, offset=0)) == PartPosition( + index=0, offset=2 + ), """This could also be considered PartPosition(index=1, offset=0) + but either way should work. """ + assert ppt.translate( + LinePosition(line=2, offset=len(ph_config.make_placeholder(0))) + ) == PartPosition( + index=2, offset=0 + ), """This must be the start of the following string because we can't + know the offset of the actual interpolation content. Ie. It cannot be + index=1 with "some" offset.""" + + def test_case_tailing_string_starts_with_newline(self, ph_config): + ppt = make_ppt(t"a{0}\nb", ph_config) + assert ppt.translate(LinePosition(line=2, offset=0)) == PartPosition( + index=2, offset=1 + ), "line 2 should be inside the tailing string" + assert ppt.translate( + LinePosition(line=1, offset=1 + len(ph_config.make_placeholder(0))) + ) == PartPosition(index=2, offset=0), ( + "end of line 1 should be inside the tailing string?" + ) + + def test_case_interpolation_without_lines(self, ph_config): + ppt = make_ppt(t"a{0}b", ph_config) + assert ppt.translate(LinePosition(line=1, offset=1)) == PartPosition( + index=1, offset=0 + ), "end of the head string is the start of the interpolation" + assert ppt.translate( + LinePosition(line=1, offset=1 + len(ph_config.make_placeholder(0))) + ) == PartPosition(index=2, offset=0), ( + "the end of the interpolation is the start of the tailing string" + ) + assert ppt.translate( + LinePosition(line=1, offset=1 + len(ph_config.make_placeholder(0)) + 1) + ) == PartPosition(index=2, offset=1), ( + "the end of the tailing string remains the end." + ) + + def test_offset_without_line(self, ph_config): ppt = make_ppt(t"a*", ph_config) assert ppt.translate(LinePosition(line=1, offset=1)) == PartPosition( index=0, offset=1 - ) + ), "the offset matches up without lines" + assert ppt.translate(LinePosition(line=1, offset=2)) == PartPosition( + index=0, offset=2 + ), "end of line is end of string" + with pytest.raises(ValueError, match="Offset exceeds reachable"): + # only 0, 1 and 2 are valid offsets for line 1 + _ = ppt.translate(LinePosition(line=1, offset=3)) == PartPosition( + index=0, offset=3 + ) - def test_line(self, ph_config): + def test_offset_with_line(self, ph_config): ppt = make_ppt(t"ab\n*", ph_config) assert ppt.translate(LinePosition(line=2, offset=0)) == PartPosition( index=0, offset=3 - ) + ), "2nd line starts at offset in the head string" + assert ppt.translate(LinePosition(line=1, offset=2)) == PartPosition( + index=0, offset=2 + ), "end of 1st line is offset to NL" + with pytest.raises(ValueError, match="Offset exceeds reachable"): + # only 0, 1 and 2 are valid offsets for line 1 + _ = ppt.translate(LinePosition(line=1, offset=3)) - def test_interpolation_after(self, ph_config): - ppt = make_ppt(t"ab\nc{0}d*", ph_config) - offset = len("".join(("c", ph_config.make_placeholder(0), "d"))) - assert ppt.translate(LinePosition(line=2, offset=offset)) == PartPosition( - index=2, offset=1 + def test_empty_strings(self, ph_config): + ppt = make_ppt(t"{0}", ph_config) + assert ppt.translate(LinePosition(line=1, offset=0)) == PartPosition( + index=1, offset=0 + ), "start of head string is start of interpolation" + assert ppt.translate( + LinePosition(line=1, offset=len(ph_config.make_placeholder(0))) + ) == PartPosition(index=2, offset=0), ( + "end of interpolation is start of tail string" ) + with pytest.raises(ValueError, match="Offset exceeds reachable"): + # Cannot go past end of the template. + _ = ppt.translate( + LinePosition(line=1, offset=len(ph_config.make_placeholder(0)) + 1) + ) - def test_interpolation_right_after(self, ph_config): - ppt = make_ppt(t"ab\nc{0}*", ph_config) - offset = len("".join(("c", ph_config.make_placeholder(0)))) - assert ppt.translate(LinePosition(line=2, offset=offset)) == PartPosition( - index=2, offset=0 + def test_empty_string(self, ph_config): + ppt = make_ppt(t"", ph_config) + assert ppt.translate(LinePosition(line=1, offset=0)) == PartPosition( + index=0, offset=0 ) + with pytest.raises(ValueError, match="Offset exceeds reachable"): + # Cannot go past end of the template. + _ = ppt.translate(LinePosition(line=1, offset=1)) - def test_interpolation_end_of_line_start_of_line(self, ph_config): - ppt = make_ppt( - t"""ab\nc{0}a\n -""", - ph_config, - ) - assert ppt.translate(LinePosition(line=3, offset=0)) == PartPosition( - index=2, offset=2 + def test_empty_line(self, ph_config): + ppt = make_ppt(t"\n", ph_config) + assert ppt.translate(LinePosition(line=1, offset=0)) == PartPosition( + index=0, offset=0 ) + with pytest.raises(ValueError, match="Offset exceeds reachable"): + # line 1 is empty, cannot offset anything + _ = ppt.translate(LinePosition(line=1, offset=1)) + assert ppt.translate(LinePosition(line=2, offset=0)) == PartPosition( + index=0, offset=1 + ), "To skip over empty line just skip over newline" + with pytest.raises(ValueError, match="Offset exceeds reachable"): + # line 2 is empty, cannot offset anything + _ = ppt.translate(LinePosition(line=2, offset=1)) + + def test_bad_parser_pos_check_bounds(self, ph_config): + ppt = make_ppt(t"abc\ndef", ph_config) + + with pytest.raises(ValueError, match="Line does not exist"): + _ = ppt.translate(LinePosition(line=3, offset=0)) + with pytest.raises(ValueError, match="Unreachable line number"): + _ = ppt.translate(LinePosition(line=0, offset=0)) + with pytest.raises(ValueError, match="Unreachable offset"): + _ = ppt.translate(LinePosition(line=1, offset=-1)) + with pytest.raises(ValueError, match="Unreachable offset"): + _ = ppt.translate(LinePosition(line=2, offset=-1)) + with pytest.raises(ValueError, match="Offset exceeds reachable"): + _ = ppt.translate(LinePosition(line=1, offset=100)) + with pytest.raises(ValueError, match="Offset exceeds reachable"): + _ = ppt.translate(LinePosition(line=2, offset=100)) + + def test_bad_parser_pos_cannot_offset_interpolation(self, ph_config): + ppt = make_ppt(t"abc\n{0}def", ph_config) + + with pytest.raises( + ValueError, + match="Invalid parser position results in offset within interpolation", + ): + _ = ppt.translate(LinePosition(line=2, offset=1)) + with pytest.raises( + ValueError, + match="Invalid parser position results in offset within interpolation", + ): + _ = ppt.translate( + LinePosition(line=2, offset=len(ph_config.make_placeholder(0)) - 1) + ) From 7bc27280e340cd28c85005f94c17f99a2eb4cca3 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Mon, 6 Jul 2026 23:12:50 -0700 Subject: [PATCH 56/78] sp --- tdom/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdom/parser.py b/tdom/parser.py index b0d7db9e..b5b4a5d0 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -148,7 +148,7 @@ def get_strings_index(self) -> int: return self.index // 2 else: raise AssertionError( - f"Index {self.index} is not references an entry in strings." + f"Index {self.index} is not referencing an entry in strings." ) def get_reader(self) -> SourceReader: From ea37e696934ef645572ff820a133e60ccf49fa0d Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Mon, 6 Jul 2026 23:16:52 -0700 Subject: [PATCH 57/78] Improve messaging when tags are not closed. --- tdom/parser.py | 12 ++++++++++-- tdom/parser_test.py | 12 ++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/tdom/parser.py b/tdom/parser.py index b5b4a5d0..295d54af 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -752,8 +752,16 @@ def close(self) -> None: "Parser expects more data, is the template valid html?" ) if self.stack: - e = ParsingError("Invalid HTML structure: unclosed tags remain.") - self.run_unclosed_ambiguous_slash_checks(self.stack[-1], e) + parent = self.stack[-1] + if isinstance(parent, (OpenTElement, OpenTComponent)): + reader = source.get_reader() + starttag_repr = reader.ref_to_repr(parent.sinfo.starttag_ref) + pos_msg = reader.make_template_pos_msg(parent.source_pos) + unclosed_msg = f"unclosed tag {starttag_repr} at {pos_msg}" + else: + unclosed_msg = "unclosed tags remain" + e = ParsingError(f"Invalid HTML structure: {unclosed_msg}.") + self.run_unclosed_ambiguous_slash_checks(parent, e) raise e if not source.placeholders.is_empty: raise ParsingError("Some placeholders were never resolved.") diff --git a/tdom/parser_test.py b/tdom/parser_test.py index 929e5c59..991e01f0 100644 --- a/tdom/parser_test.py +++ b/tdom/parser_test.py @@ -217,8 +217,8 @@ def test_parse_mismatched_tags(): _ = TemplateParser.parse(t"
Mismatched
") -def test_parse_unclosed_tag(): - with pytest.raises(ParsingError, match="unclosed tags remain"): +def test_parse_unclosed_element(): + with pytest.raises(ParsingError, match="unclosed tag
"): _ = TemplateParser.parse(t"
Unclosed") @@ -492,6 +492,14 @@ def Component(): _ = TemplateParser.parse(t"") +def test_unclosed_component_tag_error(): + def Component(): + pass + + with pytest.raises(ParsingError, match="unclosed tag <{Component}>"): + _ = TemplateParser.parse(t"<{Component}>") + + def test_placeholder_collision_avoidance(): config = make_placeholder_config() # This test is to ensure that our placeholder detection avoids collisions From ccfb1517c8afaf45236ccd36eab2225cc82ce60c Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Tue, 7 Jul 2026 21:50:42 -0700 Subject: [PATCH 58/78] Update comment. --- tdom/processor.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tdom/processor.py b/tdom/processor.py index 083a04e9..55d93658 100644 --- a/tdom/processor.py +++ b/tdom/processor.py @@ -676,8 +676,7 @@ def process( raise except Exception as e: # Causes: - # - Could be a failed "callable" formatter - # - Could be a failed "__html__()" call -- I think? # @TODO: + # - Could be a failed "callback" formatter # raise AttributeProcessingError( "Error occurred processing component attributes" From 0d3663fec27392b68138e4ea19ca31ef7789ffd0 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Tue, 7 Jul 2026 21:52:26 -0700 Subject: [PATCH 59/78] Expand component invocation testing. --- tdom/processor_test.py | 42 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/tdom/processor_test.py b/tdom/processor_test.py index 11fd9154..dea43910 100644 --- a/tdom/processor_test.py +++ b/tdom/processor_test.py @@ -1835,14 +1835,38 @@ def AttributeTypeComponent( class TestComponentErrors: def test_component_non_callable_fails(self): - with pytest.raises(ComponentInvocationError): + with pytest.raises(ComponentInvocationError, match="must be callable"): _ = html(t"<{'not a function'} />") + def test_catchall_for_attr_prep_callback_error(self): + def prep_attr(): + return 1 / 0 + + def Repeat(count: int = 0, children: Template = t"") -> Template: + return sum([children] * count, t"") + + with pytest.raises( + AttributeProcessingError, + match="Error occurred processing component attributes", + ): + _ = html(t"<{Repeat} count={prep_attr:callback}>OK") + + def test_normal_attr_error(self): + def Comp(children: Template, **kwargs) -> Template: + return t"
{children}
" + + with pytest.raises( + AttributeProcessingError, match="Cannot use int as value for aria attribute" + ): + _ = html(t"<{Comp} aria={0}>OK") + def test_component_requiring_positional_arg_fails(self): def RequiresPositional(whoops: int, /) -> Template: # pragma: no cover return t"

Positional arg: {whoops}

" - with pytest.raises(ComponentInvocationError): + with pytest.raises( + ComponentInvocationError, match="cannot have required positional arguments" + ): _ = html(t"<{RequiresPositional} />") def test_mismatched_component_closing_tag_fails(self): @@ -1852,7 +1876,9 @@ def OpenTag(children: Template) -> Template: def CloseTag(children: Template) -> Template: return t"
close
" - with pytest.raises(ComponentInvocationError): + with pytest.raises( + ComponentInvocationError, match="must match component callable" + ): _ = html(t"<{OpenTag}>Hello") @pytest.mark.parametrize( @@ -2226,7 +2252,12 @@ def test_mathml(): ) -class BadHTMLDunder: +@pytest.fixture +def bad_html_dunder(): + return _BadHTMLDunder() + + +class _BadHTMLDunder: def __html__(self): raise ValueError("bad value") @@ -2241,9 +2272,8 @@ def test_attr_error_has_matching_tnode(self): tnode = exc_info.value.template_e_states[0].tnode assert tnode and isinstance(tnode, TElement) and tnode.tag == "div" - def test_text_error_has_matching_tnode(self): + def test_text_error_has_matching_tnode(self, bad_html_dunder): "TextProcessingError should point to tnode where error first occurred." - bad_html_dunder = BadHTMLDunder() # __html__ raises exception invalid_t = t"
{bad_html_dunder}
" with pytest.raises(TextProcessingError) as exc_info: _ = html(invalid_t) From 8047cda72090f83adf9315677c8f948b614393b5 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Tue, 7 Jul 2026 22:05:26 -0700 Subject: [PATCH 60/78] Test internal function/factory component error. --- tdom/processor_test.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tdom/processor_test.py b/tdom/processor_test.py index dea43910..ea17d1a4 100644 --- a/tdom/processor_test.py +++ b/tdom/processor_test.py @@ -1881,6 +1881,34 @@ def CloseTag(children: Template) -> Template: ): _ = html(t"<{OpenTag}>Hello") + def test_func_comp_error(self): + def RaisesValueError(children: Template) -> Template: + raise ValueError("Failed to build template.") + + with pytest.raises( + ComponentInvocationError, match="Failed when invoking component callable[.]" + ) as exc_info: + _ = html(t"<{RaisesValueError}>Hello") + assert isinstance(exc_info.value.__cause__, ValueError), ( + "Original error should be chained." + ) + + def test_factory_comp_error(self): + def RaisesValueError(children: Template) -> Callable[[], Template]: + def _RaisesValueError() -> Template: + raise ValueError("Failed to build template.") + + return _RaisesValueError + + with pytest.raises( + ComponentInvocationError, + match="Failed when invoking component callable the second time.", + ) as exc_info: + _ = html(t"<{RaisesValueError}>Hello") + assert isinstance(exc_info.value.__cause__, ValueError), ( + "Original error should be chained." + ) + @pytest.mark.parametrize( "bad_value", ("", "text", None, 1, ("tuple", "of", "strs")) ) From f92b10d1448ca972546911fd73b19229c2d45092 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Tue, 7 Jul 2026 22:12:24 -0700 Subject: [PATCH 61/78] Check internal element attribute error. --- tdom/processor_test.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tdom/processor_test.py b/tdom/processor_test.py index ea17d1a4..91985997 100644 --- a/tdom/processor_test.py +++ b/tdom/processor_test.py @@ -1025,6 +1025,19 @@ def add(a, b): + Template(f"") ) + def test_callback_internal_error(self): + def raise_value_error(): + raise ValueError("Failed to compute count.") + + with pytest.raises( + AttributeProcessingError, + match="Unexpected error occurred while processing element attrs", + ) as exc_info: + _ = html(t"
") + assert isinstance(exc_info.value.__cause__, ValueError), ( + "Original error should be chained." + ) + # -------------------------------------------------------------------------- # Conditional rendering and control flow From f77723d15c9ba1c471ee32df5f11607b6cde6616 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Tue, 7 Jul 2026 22:27:27 -0700 Subject: [PATCH 62/78] Convert unified index to normal interpolations index. --- tdom/template_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdom/template_utils.py b/tdom/template_utils.py index ccbddece..0b8c2880 100644 --- a/tdom/template_utils.py +++ b/tdom/template_utils.py @@ -120,7 +120,7 @@ def slice( ) else: # @NOTE: No offset OR limit applied to interpolations. - return TemplateRef(strings=("", ""), i_indexes=(first,)) + return TemplateRef(strings=("", ""), i_indexes=((first - 1) // 2,)) else: if first % 2 == 0: strings.append(self.strings[first // 2][offset:]) From cfacc403b38ea770a71c73f697f5efb277b419cb Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Tue, 7 Jul 2026 22:27:50 -0700 Subject: [PATCH 63/78] Expand test coverage for TemplateRef.slice. --- tdom/template_utils_test.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tdom/template_utils_test.py b/tdom/template_utils_test.py index 7d65d1fe..de7a69fc 100644 --- a/tdom/template_utils_test.py +++ b/tdom/template_utils_test.py @@ -174,3 +174,30 @@ def test_newlines(self): slice_to_tref(t"
\n{0}
", start=PartPosition(index=0, offset=7)) ) assert parts == [0, "
"] + + def test_start_stop_just_interpolation(self): + parts = list( + TemplateRef.from_naive_template(t"
{0}={1}
").slice( + start=PartPosition(index=1, offset=0), + stop=PartPosition(index=2, offset=0), + ) + ) + assert parts == [0] + + def test_start_stop_just_string(self): + parts = list( + TemplateRef.from_naive_template(t"
{0}={1}
").slice( + start=PartPosition(index=2, offset=0), + stop=PartPosition(index=3, offset=0), + ) + ) + assert parts == ["="] + + def test_start_stop_substring(self): + parts = list( + TemplateRef.from_naive_template(t"
{0}={1}
").slice( + start=PartPosition(index=0, offset=1), + stop=PartPosition(index=0, offset=4), + ) + ) + assert parts == ["div"] From fd35ec7b6209eb7527abe95b9fcd302af4142331 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Tue, 7 Jul 2026 22:44:58 -0700 Subject: [PATCH 64/78] Add tests for part position to line position conversion --- tdom/source_test.py | 48 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 tdom/source_test.py diff --git a/tdom/source_test.py b/tdom/source_test.py new file mode 100644 index 00000000..1e0aa4f1 --- /dev/null +++ b/tdom/source_test.py @@ -0,0 +1,48 @@ +from .source import LinePosition, SourceReader +from .template_utils import PartPosition + + +class TestToTemplatePosition: + def test_origin(self): + t = t"
{'content'}
" + reader = SourceReader(template=t) + source_pos = PartPosition(index=0, offset=0) + assert reader.to_template_pos(source_pos) == LinePosition(line=1, offset=0) + + def test_offset_no_lines(self): + t = t"
{'content'}
" + reader = SourceReader(template=t) + source_pos = PartPosition(index=1, offset=0) + assert reader.to_template_pos(source_pos) == LinePosition( + line=1, offset=len(t.strings[0]) + ) + + def test_offset_full_interpolation(self): + t = t"
{''!s:lower}
" # conversion and formatspec + reader = SourceReader(template=t) + source_pos = PartPosition(index=2, offset=0) + assert reader.to_template_pos(source_pos) == LinePosition( + line=1, offset=len('
{""!s:lower}') + ) + + def test_line(self): + t = t"""
+{"content"}
""" + reader = SourceReader(template=t) + source_pos = PartPosition(index=2, offset=0) + assert reader.to_template_pos(source_pos) == LinePosition( + line=2, offset=len('{"content"}') + ) + + def test_line_in_interpolation(self): + t = t"""
+{ + ''' +content +''' + }
""" + reader = SourceReader(template=t) + source_pos = PartPosition(index=2, offset=0) + assert reader.to_template_pos(source_pos) == LinePosition( + line=4, offset=len('"""}') + ) From b0ff3d7f57e2c3a66f67635caab43ec94ab3732c Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Tue, 7 Jul 2026 22:51:55 -0700 Subject: [PATCH 65/78] Disable formatter for whitespace tests. --- tdom/source_test.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tdom/source_test.py b/tdom/source_test.py index 1e0aa4f1..2832787c 100644 --- a/tdom/source_test.py +++ b/tdom/source_test.py @@ -26,8 +26,11 @@ def test_offset_full_interpolation(self): ) def test_line(self): + # whitespace is part of test + # fmt: off t = t"""
{"content"}
""" + # fmt: on reader = SourceReader(template=t) source_pos = PartPosition(index=2, offset=0) assert reader.to_template_pos(source_pos) == LinePosition( @@ -35,14 +38,15 @@ def test_line(self): ) def test_line_in_interpolation(self): + # whitespace is part of test + # fmt: off t = t"""
-{ - ''' +{''' content -''' - }
""" +'''}
""" + # fmt: on reader = SourceReader(template=t) source_pos = PartPosition(index=2, offset=0) assert reader.to_template_pos(source_pos) == LinePosition( - line=4, offset=len('"""}') + line=4, offset=len("'''}") ) From b26165dd80787d78e6cab04b10904dd7dda4d6f7 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Sun, 12 Jul 2026 13:50:54 -0700 Subject: [PATCH 66/78] Remove dependency on the actively tracked template index. --- tdom/parser.py | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/tdom/parser.py b/tdom/parser.py index 295d54af..50d6cf36 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -143,14 +143,6 @@ def __next__(self): else: raise StopIteration - def get_strings_index(self) -> int: - if self.index % 2 == 0: - return self.index // 2 - else: - raise AssertionError( - f"Index {self.index} is not referencing an entry in strings." - ) - def get_reader(self) -> SourceReader: return SourceReader(template=self.template) @@ -313,20 +305,20 @@ def make_open_tag( # relying on higher layers to validate types and render correctly. i_index = tag_ref.i_indexes[0] - # @NOTE: This must be stored when the tag is handled since it is - # set based on when the template parts are fed in and otherwise - # might be out of sync. + # @NOTE: This must be called when the tag is handled since it is + # populated based on the most recently finished start tag. Otherwise + # the value will be out of sync. + starttag_ref = self.get_starttag_ref() # The starting s_index of the component's children template. Note that # this string either contains ">" or " />". It might not be # i_index + 1 because attributes WITHIN the component's tag might # contain interpolations causing the i_index (and s_index) to advance # arbitrarily. - children_start_s_index = self.get_source().get_strings_index() - - # @NOTE: This must be called when the tag is handled since it is - # populated based on the most recently finished start tag. Otherwise - # the value will be out of sync. - starttag_ref = self.get_starttag_ref() + children_start_s_index = ( + i_index # i_index of comp callable, from start of the WHOLE template + + len(starttag_ref.strings) # then count up to the end + - 1 # remove 1 since we want an index instead of a limit + ) # @NOTE: The last string should terminate the starttag and end with ">" # So this length is the offset from the last interpolation to the start # of the children's leading string. From 64b668d42145405255b3cb42b9396fdcfd416cbe Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Sun, 12 Jul 2026 14:10:23 -0700 Subject: [PATCH 67/78] Fix ty warning for now. --- tdom/parser.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tdom/parser.py b/tdom/parser.py index 50d6cf36..0073d0f5 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -363,6 +363,9 @@ def finalize_tag( children=tuple(children), source_pos=source_pos, ) + source_pos = ( + open_tag.source_pos + ) # Re-assignment for ty regression in 0.0.59 self.sinfo_table[source_pos] = sinfo.close(endtag_pos=endtag_pos) case OpenTFragment(children=children, source_pos=source_pos): tnode = TFragment(children=tuple(children), source_pos=source_pos) From e2f55fb3fda174fdc7811f0cdaf6dfca6b23b9d1 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Tue, 14 Jul 2026 23:09:35 -0700 Subject: [PATCH 68/78] Drop ref attrs until we need them. --- tdom/parser.py | 73 ++++++++++++++++++++++---------------------------- tdom/tnodes.py | 2 -- 2 files changed, 32 insertions(+), 43 deletions(-) diff --git a/tdom/parser.py b/tdom/parser.py index 0073d0f5..ca301b25 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -55,14 +55,10 @@ class OpenTagSourceInfo: @NOTE: This is an temporary structure that will be finalized when the tag is closed. - - @TODO: Do we need `ref_attrs` or should we just try to get by with the tattrs? """ starttag_ref: TemplateRef " Entire starttag as parsed except placeholders are replaced by references. " - ref_attrs: tuple[tuple[TemplateRef, TemplateRef | None], ...] - " Attrs as parsed except placeholders are replaced by references. " startend: bool " Was parsed as startend tag, ie. . " starttag_pos: PartPosition @@ -71,7 +67,6 @@ class OpenTagSourceInfo: def close(self, endtag_pos: PartPosition | None = None) -> TagSourceInfo: return TagSourceInfo( starttag_ref=self.starttag_ref, - ref_attrs=self.ref_attrs, startend=self.startend, starttag_pos=self.starttag_pos, endtag_pos=endtag_pos, @@ -256,20 +251,6 @@ def make_tattrs(self, attrs: Sequence[HTMLAttribute]) -> tuple[TAttribute, ...]: """Build TAttributes from raw attribute tuples.""" return tuple(self.make_tattr(attr) for attr in attrs) - def make_ref_attr( - self, source: SourceTracker, attr: HTMLAttribute - ) -> tuple[TemplateRef, TemplateRef | None]: - return ( - source.find_placeholders(attr[0]), - source.find_placeholders(attr[1]) if attr[1] is not None else None, - ) - - def make_ref_attrs( - self, attrs: Sequence[HTMLAttribute] - ) -> tuple[tuple[TemplateRef, TemplateRef | None], ...]: - source = self.get_source() - return tuple(self.make_ref_attr(source, attr) for attr in attrs) - # ------------------------------------------ # Tag Helpers # ------------------------------------------ @@ -287,7 +268,6 @@ def make_open_tag( attrs=self.make_tattrs(attrs), sinfo=OpenTagSourceInfo( starttag_ref=self.get_starttag_ref(), - ref_attrs=self.make_ref_attrs(attrs), startend=startend, starttag_pos=source_pos, ), @@ -334,7 +314,6 @@ def make_open_tag( source_pos=source_pos, sinfo=OpenTagSourceInfo( starttag_ref=starttag_ref, - ref_attrs=self.make_ref_attrs(attrs), startend=startend, starttag_pos=source_pos, ), @@ -450,6 +429,7 @@ def extract_component_children_ref( def make_mismatch_error( self, starttag_sinfo: OpenTagSourceInfo, + starttag_attrs: tuple[TAttribute, ...], endtag_ref: TemplateRef, endtag_pos: PartPosition, ) -> ParsingError: @@ -461,7 +441,7 @@ def make_mismatch_error( e = ParsingError( f"Mismatched closing tag at {endtag_pos_msg} for {starttag_repr} at {starttag_pos_msg}." ) - if self.has_ambiguous_forward_slash(starttag_sinfo): + if self.has_ambiguous_forward_slash(starttag_sinfo, starttag_attrs): e.add_note( f'Did you mean to quote the last attribute or put a space before "/>" for "{starttag_repr}" at {starttag_pos_msg}?' ) @@ -486,7 +466,7 @@ def validate_end_tag(self, tag: str, open_tag: OpenTag) -> int | None: case OpenTElement(): if tag_ref.is_singleton or (tag_ref.is_literal and tag != open_tag.tag): raise self.make_mismatch_error( - open_tag.sinfo, tag_ref, self.get_source_pos() + open_tag.sinfo, open_tag.attrs, tag_ref, self.get_source_pos() ) elif not tag_ref.is_singleton and not tag_ref.is_literal: raise self.make_invalid_endtag_error(tag_ref, self.get_source_pos()) @@ -496,7 +476,7 @@ def validate_end_tag(self, tag: str, open_tag: OpenTag) -> int | None: case OpenTComponent(): if tag_ref.is_literal: raise self.make_mismatch_error( - open_tag.sinfo, tag_ref, self.get_source_pos() + open_tag.sinfo, open_tag.attrs, tag_ref, self.get_source_pos() ) elif not tag_ref.is_singleton: raise self.make_invalid_endtag_error(tag_ref, self.get_source_pos()) @@ -517,7 +497,9 @@ def get_starttag_ref(self) -> TemplateRef: return self.get_source().find_placeholders(starttag_text) def has_ambiguous_forward_slash( - self, sinfo: OpenTagSourceInfo | TagSourceInfo | None + self, + sinfo: OpenTagSourceInfo | TagSourceInfo | None, + attrs: tuple[TAttribute, ...], ) -> bool: """ Detect when an unquoted attribute value consumes a trailing "/" that @@ -530,20 +512,29 @@ def has_ambiguous_forward_slash( Or more often "<{Component} title={title}/>" which should be corrected with "<{Component} title={title} />". """ - if sinfo is not None: - return ( - # has attributes - len(sinfo.ref_attrs) > 0 - # last attr not bare attribute - and sinfo.ref_attrs[-1][1] is not None - # last char of last string of value of last ref attr is "/" - and sinfo.ref_attrs[-1][1].strings[-1][-1] == "/" - # parsed starttag ends with "/>" - and sinfo.starttag_ref.strings[-1].endswith("/>") - # if parsed as startend then its not ambiguous - and not sinfo.startend + return ( + # has source info + sinfo is not None + # has attributes + and len(attrs) > 0 + # last attribute ends with "/" + # @NOTE: spread and interpolated attrs never do + and ( + ( + isinstance(attrs[-1], TLiteralAttribute) + and attrs[-1].value is not None + and attrs[-1].value.endswith("/") + ) + or ( + isinstance(attrs[-1], TTemplatedAttribute) + and attrs[-1].value_ref.strings[-1].endswith("/") + ) ) - return False + # parsed starttag ends with "/>", + and sinfo.starttag_ref.strings[-1].endswith("/>") + # if parsed AS startend already then its not ambiguous + and not sinfo.startend + ) # ------------------------------------------ # HTMLParser tag callbacks @@ -673,7 +664,7 @@ def run_unclosed_ambiguous_slash_checks( reader = source.get_reader() if isinstance( parent, (OpenTElement, OpenTComponent) - ) and self.has_ambiguous_forward_slash(parent.sinfo): + ) and self.has_ambiguous_forward_slash(parent.sinfo, parent.attrs): # CASE: "<{C1} attr={value}/>" -- maybe user meant to self-close? # CASE: "
" -- mayber user meant to self-close? starttag_ref = parent.sinfo.starttag_ref @@ -695,7 +686,7 @@ def run_unclosed_ambiguous_slash_checks( if child.source_pos is not None else None ) - if sinfo and self.has_ambiguous_forward_slash(sinfo): + if sinfo and self.has_ambiguous_forward_slash(sinfo, child.attrs): full_starttag_repr = reader.ref_to_repr(sinfo.starttag_ref) e.add_note( f'Did you mean to quote the last attribute or put a space before "/>" for "{full_starttag_repr}"?' @@ -728,7 +719,7 @@ def run_unclosed_ambiguous_slash_checks( if comp.source_pos is not None else None ) - if sinfo and self.has_ambiguous_forward_slash(sinfo): + if sinfo and self.has_ambiguous_forward_slash(sinfo, comp.attrs): full_starttag_repr = reader.ref_to_repr(sinfo.starttag_ref) e.add_note( f'Did you mean to quote the last attribute or put a space before "/>" for "{full_starttag_repr}"?' diff --git a/tdom/tnodes.py b/tdom/tnodes.py index 1b375e8b..3e8392c8 100644 --- a/tdom/tnodes.py +++ b/tdom/tnodes.py @@ -119,8 +119,6 @@ class TagSourceInfo: starttag_ref: TemplateRef " Entire starttag as parsed except placeholders are replaced by references. " - ref_attrs: tuple[tuple[TemplateRef, TemplateRef | None], ...] - " Attrs as parsed except placeholders are replaced by references. " startend: bool " Was parsed as startend tag, ie. . " starttag_pos: PartPosition From 657e7d1375825cd5921c7d260e6e1761d53b9a57 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Sat, 25 Jul 2026 21:31:56 -0700 Subject: [PATCH 69/78] Fix first==last strings bug and interpolations bug, improve clarity. --- tdom/template_utils.py | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/tdom/template_utils.py b/tdom/template_utils.py index 0b8c2880..a5e132ad 100644 --- a/tdom/template_utils.py +++ b/tdom/template_utils.py @@ -103,6 +103,18 @@ def slice( """ Slice template ref based on the given start and stop. """ + # @NOTE: A start interpolation must always be defined since start == None + # will be the first "part" which is a string (index=0). + if start and start.index % 2 != 0: + assert start.offset == 0, ( + "Interpolation part positions must always have offset 0." + ) + # @NOTE: A stop interpolation must always be defined since stop == None + # will be the last "part" which is a string (index=size - 1). + if stop and stop.index % 2 != 0: + assert stop.offset == 0, ( + "Interpolation part positions must always have offset 0." + ) size = 2 * len(self.strings) - 1 first = start.index if start and start.index is not None else 0 assert 0 <= first < size @@ -115,17 +127,19 @@ def slice( i_indexes = [] if first == last: if first % 2 == 0: - return TemplateRef( - strings=(self.strings[first][offset:limit],), i_indexes=() - ) + strings.append(self.strings[first // 2][offset:limit]) else: - # @NOTE: No offset OR limit applied to interpolations. - return TemplateRef(strings=("", ""), i_indexes=((first - 1) // 2,)) + # offset == 0, so this is the equivalent of an empty interval + # therefore we should exclude this interpolation but + # template-ify with empty string. + strings.append("") + return TemplateRef(strings=tuple(strings), i_indexes=tuple(i_indexes)) else: if first % 2 == 0: strings.append(self.strings[first // 2][offset:]) else: - # @NOTE: No offset applied to interpolations. + # offset == 0, so template-ify with empty string but start by + # including this interpolation. strings.append("") i_indexes.append((first - 1) // 2) @@ -136,8 +150,9 @@ def slice( else: strings.append(self.strings[index // 2]) else: - # @NOTE: No limit applied to interpolations. - if index != last: + if index == last: + break # offset == 0, so exclude this interpolation. + else: i_indexes.append((index - 1) // 2) return TemplateRef(strings=tuple(strings), i_indexes=tuple(i_indexes)) From fd07cae5a6e862577956f9ccdf77085a35a3a9c9 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Sun, 26 Jul 2026 14:12:20 -0700 Subject: [PATCH 70/78] Import from true origin. --- tdom/parser.py | 3 +-- tdom/parser_test.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tdom/parser.py b/tdom/parser.py index ca301b25..57475b36 100644 --- a/tdom/parser.py +++ b/tdom/parser.py @@ -14,10 +14,9 @@ ) from .source import ( LinePosition, - PartPosition, SourceReader, ) -from .template_utils import TemplateRef, combine_template_refs +from .template_utils import PartPosition, TemplateRef, combine_template_refs from .tnodes import ( TagSourceInfo, TAttribute, diff --git a/tdom/parser_test.py b/tdom/parser_test.py index 991e01f0..46fc5ff0 100644 --- a/tdom/parser_test.py +++ b/tdom/parser_test.py @@ -8,8 +8,7 @@ TemplateParser, ) from .placeholders import make_placeholder_config -from .source import PartPosition -from .template_utils import TemplateRef +from .template_utils import PartPosition, TemplateRef from .tnodes import ( TComment, TComponent, From 2a1ef8cff030af3c7f03b10b42846061a5c0facd Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Sun, 26 Jul 2026 14:14:58 -0700 Subject: [PATCH 71/78] Move unified index handling into template_utils. --- tdom/parser_utils.py | 10 ++-------- tdom/parser_utils_test.py | 4 ++-- tdom/template_utils.py | 18 ++++++++++++++++++ 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/tdom/parser_utils.py b/tdom/parser_utils.py index 123b561a..3b93ac54 100644 --- a/tdom/parser_utils.py +++ b/tdom/parser_utils.py @@ -3,7 +3,7 @@ from .placeholders import PlaceholderConfig from .source import LinePosition, MutableLinePosition -from .template_utils import PartPosition +from .template_utils import PartPosition, validate_part_position type HTMLAttribute = tuple[str, str | None] @@ -53,13 +53,7 @@ def validate(self, parser_pos: LinePosition): def translate(self, parser_pos: LinePosition) -> PartPosition: self.validate(parser_pos) part_pos = parser_pos_to_part_pos(self.source_text_parts, parser_pos) - if part_pos.index % 2 != 0 and part_pos.offset != 0: - # You can only land on the start of an interpolation - # There is no way to translate a position within a placeholder - # to a position within the original interpolation representation. - raise ValueError( - "Invalid parser position results in offset within interpolation!" - ) + validate_part_position(part_pos) return part_pos diff --git a/tdom/parser_utils_test.py b/tdom/parser_utils_test.py index 03f9befb..87f258db 100644 --- a/tdom/parser_utils_test.py +++ b/tdom/parser_utils_test.py @@ -147,12 +147,12 @@ def test_bad_parser_pos_cannot_offset_interpolation(self, ph_config): with pytest.raises( ValueError, - match="Invalid parser position results in offset within interpolation", + match="Invalid part position, interpolations are not divisible, offset must be 0.", ): _ = ppt.translate(LinePosition(line=2, offset=1)) with pytest.raises( ValueError, - match="Invalid parser position results in offset within interpolation", + match="Invalid part position, interpolations are not divisible, offset must be 0.", ): _ = ppt.translate( LinePosition(line=2, offset=len(ph_config.make_placeholder(0)) - 1) diff --git a/tdom/template_utils.py b/tdom/template_utils.py index a5e132ad..c68ccb10 100644 --- a/tdom/template_utils.py +++ b/tdom/template_utils.py @@ -193,3 +193,21 @@ class PartPosition: offset: int = 0 " Offset from the start of the template part. " + + +def validate_part_position(part_pos: PartPosition) -> None: + """ + Basic part position validation for parts that are converted to template + source `LinePosition`. + + @TODO: This might move into the constructor eventually depending on usage. + """ + if part_pos.index % 2 != 0 and part_pos.offset != 0: + # You can only land on the start of an interpolation + raise ValueError( + "Invalid part position, interpolations are not divisible, offset must be 0." + ) + if not (part_pos.offset >= 0): + raise ValueError("Offset must always be positive or zero.") + if not (part_pos.index >= 0): + raise ValueError("Index must always be positive or zero.") From 3220d37dff4ad1e3716371fce90652052c685831 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Sun, 26 Jul 2026 14:34:08 -0700 Subject: [PATCH 72/78] Correct test, use function. --- tdom/template_utils_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tdom/template_utils_test.py b/tdom/template_utils_test.py index de7a69fc..12699c41 100644 --- a/tdom/template_utils_test.py +++ b/tdom/template_utils_test.py @@ -132,16 +132,16 @@ def test_string_only_start_stop(self): assert parts == ["><"] def test_single_interpolation_stop(self): - parts = TemplateRef.from_naive_template(t"
{0}
").slice( - start=None, stop=PartPosition(index=1, offset=0) + parts = slice_to_tref( + t"
{0}
", start=None, stop=PartPosition(index=1, offset=0) ) assert list(parts) == ["
"] def test_single_interpolation_start(self): parts = slice_to_tref( - t"
{0}
", start=None, stop=PartPosition(index=1, offset=0) + t"
{0}
", start=PartPosition(index=1, offset=0) ) - assert list(parts) == ["
"] + assert list(parts) == [0, "
"] def test_end_after_interpolation(self): parts = list( From cecb384c0efca6d898bf14c077574a30b9ae37bc Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Sun, 26 Jul 2026 14:50:02 -0700 Subject: [PATCH 73/78] Use unified indexing in tref iter and group iter tests. --- tdom/template_utils.py | 22 ++++++++----- tdom/template_utils_test.py | 66 ++++++++++++++++++------------------- 2 files changed, 45 insertions(+), 43 deletions(-) diff --git a/tdom/template_utils.py b/tdom/template_utils.py index c68ccb10..3dd25088 100644 --- a/tdom/template_utils.py +++ b/tdom/template_utils.py @@ -80,15 +80,19 @@ def __post_init__(self): ) def __iter__(self): - index = 0 - last_s_index = len(self.strings) - 1 - while index <= last_s_index: - s = self.strings[index] - if s: - yield s - if index < last_s_index: - yield self.i_indexes[index] - index += 1 + """ + Yield parts like `string.templatelib.Template`: `str, [int, str], ...`. + + Empty strings are omitted which parallels the behavior + of `Template.__iter__`. Use `parts_iter` to include empty strings. + """ + size = len(self.strings) * 2 - 1 + for index in range(size): + if index % 2 == 0: + if (s := self.strings[index//2]): + yield s + else: + yield self.i_indexes[(index-1)//2] def resolve(self, interpolations: tuple[Interpolation, ...]) -> Template: """Use the given interpolations to resolve this reference template into a Template.""" diff --git a/tdom/template_utils_test.py b/tdom/template_utils_test.py index 12699c41..13a440d7 100644 --- a/tdom/template_utils_test.py +++ b/tdom/template_utils_test.py @@ -2,7 +2,13 @@ import pytest -from .template_utils import TemplateRef, combine_template_refs, template_from_parts +from .template_utils import ( + PartPosition, + TemplateRef, + combine_template_refs, + slice_to_tref, + template_from_parts, +) def test_template_from_parts() -> None: @@ -57,40 +63,37 @@ def test_combine_template_refs(): ) -def test_template_ref_iter_singleton(): - assert list(TemplateRef.from_naive_template(t"{1}")) == [1] +class TestTRefIter: + "Tests for TemplateRef.__iter__." + def test_template_ref_iter_singleton(self): + assert list(TemplateRef.from_naive_template(t"{1}")) == [1] -def test_template_ref_iter_empty(): - assert list(TemplateRef.from_naive_template(t"")) == [] + def test_template_ref_iter_empty(self): + assert list(TemplateRef.from_naive_template(t"")) == [] + def test_template_ref_iter_empty_prefix(self): + assert list(TemplateRef.from_naive_template(t"{1}def")) == [1, "def"] -def test_template_ref_iter_empty_prefix(): - assert list(TemplateRef.from_naive_template(t"{1}def")) == [1, "def"] + def test_template_ref_iter_empty_suffix(self): + assert list(TemplateRef.from_naive_template(t"abc{1}")) == ["abc", 1] + def test_template_ref_iter_literal(self): + assert list(TemplateRef.from_naive_template(t"abc")) == ["abc"] -def test_template_ref_iter_empty_suffix(): - assert list(TemplateRef.from_naive_template(t"abc{1}")) == ["abc", 1] + def test_template_ref_iter_only_interpolations(self): + assert list(TemplateRef.from_naive_template(t"{1}{3}{5}")) == [1, 3, 5] - -def test_template_ref_iter_literal(): - assert list(TemplateRef.from_naive_template(t"abc")) == ["abc"] - - -def test_template_ref_iter_only_interpolations(): - assert list(TemplateRef.from_naive_template(t"{1}{3}{5}")) == [1, 3, 5] - - -def test_template_ref_iter_complete(): - assert list(TemplateRef.from_naive_template(t"abc{1}def{3}ghi{5}jkl")) == [ - "abc", - 1, - "def", - 3, - "ghi", - 5, - "jkl", - ] + def test_template_ref_iter_complete(self): + assert list(TemplateRef.from_naive_template(t"abc{1}def{3}ghi{5}jkl")) == [ + "abc", + 1, + "def", + 3, + "ghi", + 5, + "jkl", + ] def test_template_ref_resolve(): @@ -103,9 +106,6 @@ def test_template_ref_resolve(): assert resolved_t.strings == ("", "b", "d", "f") -from .template_utils import PartPosition, slice_to_tref - - class TestSliceToTRef: def test_string_only_stop(self): parts = list( @@ -138,9 +138,7 @@ def test_single_interpolation_stop(self): assert list(parts) == ["
"] def test_single_interpolation_start(self): - parts = slice_to_tref( - t"
{0}
", start=PartPosition(index=1, offset=0) - ) + parts = slice_to_tref(t"
{0}
", start=PartPosition(index=1, offset=0)) assert list(parts) == [0, "
"] def test_end_after_interpolation(self): From a02bf9f113927e0f801090c3350dcb84cbd57041 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Sun, 26 Jul 2026 14:57:28 -0700 Subject: [PATCH 74/78] Add TemplateRef.parts_iter that includes empty strings. --- tdom/template_utils.py | 15 +++++++++-- tdom/template_utils_test.py | 51 +++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/tdom/template_utils.py b/tdom/template_utils.py index 3dd25088..b76a08d7 100644 --- a/tdom/template_utils.py +++ b/tdom/template_utils.py @@ -79,6 +79,17 @@ def __post_init__(self): "TemplateRef must have one more string than interpolation indexes." ) + def parts_iter(self): + """ + Similar to __iter__ but returns empty strings. + """ + size = len(self.strings) * 2 - 1 + for index in range(size): + if index % 2 == 0: + yield self.strings[index // 2] + else: + yield self.i_indexes[(index - 1) // 2] + def __iter__(self): """ Yield parts like `string.templatelib.Template`: `str, [int, str], ...`. @@ -89,10 +100,10 @@ def __iter__(self): size = len(self.strings) * 2 - 1 for index in range(size): if index % 2 == 0: - if (s := self.strings[index//2]): + if s := self.strings[index // 2]: yield s else: - yield self.i_indexes[(index-1)//2] + yield self.i_indexes[(index - 1) // 2] def resolve(self, interpolations: tuple[Interpolation, ...]) -> Template: """Use the given interpolations to resolve this reference template into a Template.""" diff --git a/tdom/template_utils_test.py b/tdom/template_utils_test.py index 13a440d7..5bc8e788 100644 --- a/tdom/template_utils_test.py +++ b/tdom/template_utils_test.py @@ -96,6 +96,57 @@ def test_template_ref_iter_complete(self): ] +class TestTRefPartsIter: + "Tests for TemplateRef.parts_iter." + + def test_singleton(self): + assert list(TemplateRef.from_naive_template(t"{1}").parts_iter()) == ["", 1, ""] + + def test_empty(self): + assert list(TemplateRef.from_naive_template(t"").parts_iter()) == [""] + + def test_empty_prefix(self): + assert list(TemplateRef.from_naive_template(t"{1}def").parts_iter()) == [ + "", + 1, + "def", + ] + + def test_empty_suffix(self): + assert list(TemplateRef.from_naive_template(t"abc{1}").parts_iter()) == [ + "abc", + 1, + "", + ] + + def test_literal(self): + assert list(TemplateRef.from_naive_template(t"abc").parts_iter()) == ["abc"] + + def test_only_interpolations(self): + assert list(TemplateRef.from_naive_template(t"{1}{3}{5}").parts_iter()) == [ + "", + 1, + "", + 3, + "", + 5, + "", + ] + + def test_complete(self): + assert list( + TemplateRef.from_naive_template(t"abc{1}def{3}ghi{5}jkl").parts_iter() + ) == [ + "abc", + 1, + "def", + 3, + "ghi", + 5, + "jkl", + ] + + def test_template_ref_resolve(): src_t = t"{'a'}b{'c'}d{'e'}f" src_ref = TemplateRef( From d22385336084910b3461d1ec16d68209bc66eca6 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Sun, 26 Jul 2026 15:48:55 -0700 Subject: [PATCH 75/78] Expand tests to cover same start and stop for both a string and interpolation. --- tdom/template_utils_test.py | 43 ++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/tdom/template_utils_test.py b/tdom/template_utils_test.py index 5bc8e788..6688c7ec 100644 --- a/tdom/template_utils_test.py +++ b/tdom/template_utils_test.py @@ -160,8 +160,8 @@ def test_template_ref_resolve(): class TestSliceToTRef: def test_string_only_stop(self): parts = list( - TemplateRef.from_naive_template(t"
").slice( - start=None, stop=PartPosition(index=0, offset=5) + slice_to_tref( + t"
", start=None, stop=PartPosition(index=0, offset=5) ) ) assert parts == ["
"] @@ -224,9 +224,34 @@ def test_newlines(self): ) assert parts == [0, "
"] + def test_start_stop_same_string_is_empty(self): + assert slice_to_tref( + t"
{0}={1}
", + start=PartPosition(index=2, offset=0), + stop=PartPosition(index=2, offset=0), + ).is_empty + + def test_start_stop_same_interpolation_is_empty(self): + assert slice_to_tref( + t"
{0}={1}
", + start=PartPosition(index=3, offset=0), + stop=PartPosition(index=3, offset=0), + ).is_empty + + def test_start_stop_same_substring(self): + parts = list( + slice_to_tref( + t"
{0}={1}
", + start=PartPosition(index=0, offset=1), + stop=PartPosition(index=0, offset=4), + ) + ) + assert parts == ["div"] + def test_start_stop_just_interpolation(self): parts = list( - TemplateRef.from_naive_template(t"
{0}={1}
").slice( + slice_to_tref( + t"
{0}={1}
", start=PartPosition(index=1, offset=0), stop=PartPosition(index=2, offset=0), ) @@ -235,18 +260,10 @@ def test_start_stop_just_interpolation(self): def test_start_stop_just_string(self): parts = list( - TemplateRef.from_naive_template(t"
{0}={1}
").slice( + slice_to_tref( + t"
{0}={1}
", start=PartPosition(index=2, offset=0), stop=PartPosition(index=3, offset=0), ) ) assert parts == ["="] - - def test_start_stop_substring(self): - parts = list( - TemplateRef.from_naive_template(t"
{0}={1}
").slice( - start=PartPosition(index=0, offset=1), - stop=PartPosition(index=0, offset=4), - ) - ) - assert parts == ["div"] From a9520c4412513dd6a6256eccd40f266cd62ede8b Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Sun, 26 Jul 2026 15:57:35 -0700 Subject: [PATCH 76/78] Drop the walrus. --- tdom/template_utils.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tdom/template_utils.py b/tdom/template_utils.py index b76a08d7..4d44f442 100644 --- a/tdom/template_utils.py +++ b/tdom/template_utils.py @@ -99,11 +99,10 @@ def __iter__(self): """ size = len(self.strings) * 2 - 1 for index in range(size): - if index % 2 == 0: - if s := self.strings[index // 2]: - yield s - else: + if index % 2 != 0: yield self.i_indexes[(index - 1) // 2] + elif self.strings[index // 2]: + yield self.strings[index // 2] def resolve(self, interpolations: tuple[Interpolation, ...]) -> Template: """Use the given interpolations to resolve this reference template into a Template.""" From 857d2dd1be98daddb20b60e17e249fa751aa5eb2 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Sun, 26 Jul 2026 16:58:42 -0700 Subject: [PATCH 77/78] Insource combining trefs and expand tests. --- tdom/template_utils.py | 11 ++++++--- tdom/template_utils_test.py | 46 ++++++++++++++++++++++++++----------- 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/tdom/template_utils.py b/tdom/template_utils.py index 4d44f442..ac2e176a 100644 --- a/tdom/template_utils.py +++ b/tdom/template_utils.py @@ -18,10 +18,15 @@ def template_from_parts( def combine_template_refs(*template_refs: TemplateRef) -> TemplateRef: """Concatenate multiple template refs together into a single ref.""" - # trefs -> naive templates -> naive template -> tref - return TemplateRef.from_naive_template( - Template(*chain.from_iterable(tr.to_naive_template() for tr in template_refs)) + combined_strings = [""] + combined_i_indexes = tuple( + chain.from_iterable(tref.i_indexes for tref in template_refs) ) + for tref in template_refs: + # Join last tref tail to this tref head + combined_strings[-1] = combined_strings[-1] + tref.strings[0] + combined_strings.extend(tref.strings[1:]) + return TemplateRef(strings=tuple(combined_strings), i_indexes=combined_i_indexes) @dataclass(slots=True, frozen=True) diff --git a/tdom/template_utils_test.py b/tdom/template_utils_test.py index 6688c7ec..07816135 100644 --- a/tdom/template_utils_test.py +++ b/tdom/template_utils_test.py @@ -48,19 +48,39 @@ def test_template_ref_post_init_validation() -> None: _ = TemplateRef(("Hello",), (0, 1)) -def test_combine_template_refs(): - template_refs = map( - TemplateRef.from_naive_template, - [ - t"ab", - t"c{0}d", - t"ef{1}", - t"{2}ghi", - ], - ) - assert combine_template_refs(*template_refs) == TemplateRef.from_naive_template( - t"abc{0}def{1}{2}ghi" - ) +class TestCombineTemplateRefs: + def test_general_case(self): + template_refs = map( + TemplateRef.from_naive_template, + [ + t"ab", + t"c{100}d{0}e", + t"f{200}", + t"{300}ghi", + ], + ) + tref = combine_template_refs(*template_refs) + assert tref.strings == ("abc", "d", "ef", "", "ghi") and tref.i_indexes == ( + 100, + 0, + 200, + 300, + ) + + def test_strings(self): + trefs = [ + TemplateRef(strings=(s,), i_indexes=()) for s in ["ab", "", "cdef", "g", ""] + ] + tref = combine_template_refs(*trefs) + assert tref.strings == ("abcdefg",) and tref.i_indexes == () + + def test_indexes(self): + trefs = [TemplateRef(strings=("", ""), i_indexes=(i,)) for i in (100, 200, 300)] + tref = combine_template_refs(*trefs) + assert tref.strings == ("", "", "", "") and tref.i_indexes == (100, 200, 300) + + def test_null(self): + assert combine_template_refs() == TemplateRef.empty() class TestTRefIter: From f9459caece3ac00a1ade565039401e67c5f1e754 Mon Sep 17 00:00:00 2001 From: Ian Wilson Date: Mon, 27 Jul 2026 17:05:15 -0700 Subject: [PATCH 78/78] Simplify parser pos translator by precomputing and preconfiguring more away ParserPosition. --- tdom/parser_utils.py | 197 +++++++++++++++++++++++++------------- tdom/parser_utils_test.py | 8 +- 2 files changed, 138 insertions(+), 67 deletions(-) diff --git a/tdom/parser_utils.py b/tdom/parser_utils.py index 3b93ac54..4e769828 100644 --- a/tdom/parser_utils.py +++ b/tdom/parser_utils.py @@ -8,10 +8,51 @@ type HTMLAttribute = tuple[str, str | None] +@dataclass(frozen=True) +class ParserPosition: + """ + A parser position returned by the template parser. + + In certain cases the offset points at "nothing" but has extra meaning + handling by flags. These can be used when converting this position + to a PartPosition. + """ + + line: int = 1 + " Line number, starts counting at 1. " + + offset: int = 0 + " Offset into the line, starts counting at 0. " + + eol: bool = False + " Offset to the NL at the end of line. " + + eof: bool = False + " Offset to the end of the input, there is no line terminator." + + +def precompute_line_to_part_pos( + source_text_parts: tuple[str, ...], +) -> dict[int, PartPosition]: + line_to_part_pos = {1: PartPosition(0, 0)} + line = 1 + for index, part_text in enumerate(source_text_parts): + start = 0 + while 1: + nl_index = part_text.find("\n", start) + if nl_index != -1: + line += 1 + start = nl_index + 1 + line_to_part_pos[line] = PartPosition(index, start) + else: + break + return line_to_part_pos + + def make_parser_pos_translator( template: Template, config: PlaceholderConfig ) -> ParserPositionTranslator: - # Precompute these. + source_text_parts = tuple( template.strings[index // 2] if index % 2 == 0 @@ -19,7 +60,12 @@ def make_parser_pos_translator( for index in range(2 * len(template.strings) - 1) ) source_text_lines = tuple("".join(source_text_parts).split("\n")) - return ParserPositionTranslator(source_text_parts, source_text_lines) + + line_to_part_pos = precompute_line_to_part_pos(source_text_parts) + + return ParserPositionTranslator( + source_text_parts, source_text_lines, line_to_part_pos + ) @dataclass @@ -30,99 +76,118 @@ class ParserPositionTranslator: source_text_lines: tuple[str, ...] " The source text of the entire template, with placeholders. " - def validate(self, parser_pos: LinePosition): + line_to_part_pos: dict[int, PartPosition] + " Precomputed mapping from line number to part position. " + + def validate_raw_parser_pos( + self, + raw_parser_pos: LinePosition, + coerce_eol: bool = True, + coerce_eof: bool = True, + ) -> ParserPosition: """ Check parser position targets existing line and offset in template. This attempts to reduce the complexity of the translating by letting us assume the translation is possible. """ - if parser_pos.line > len(self.source_text_lines): + line = raw_parser_pos.line + offset = raw_parser_pos.offset + if line > len(self.source_text_lines): raise ValueError("Line does not exist in source.") - elif parser_pos.line <= 0: + elif line <= 0: raise ValueError("Unreachable line number, must be > 0.") - # @NOTE: This includes an offset that is at the end of the line. - last_index = len(self.source_text_lines[parser_pos.line - 1]) - if parser_pos.offset > last_index: + # either eol or eof + end_index = len(self.source_text_lines[line - 1]) + last_line = len(self.source_text_lines) # 1-based + eof = False + eol = False + if offset < 0: + raise ValueError("Unreachable offset, must be >= 0.") + elif offset == end_index and line == last_line: + if coerce_eof: + eof = True + else: + raise ValueError( + f"Offset exceeds reachable characters of last line and coerce EOF is off: {line}: {offset} == {end_index}" + ) + elif offset == end_index and line != last_line: + if coerce_eol: + eol = True + else: + raise ValueError( + f"Offset exceeds reachable characters of line terminated with newline and coerce EOL is off: {line}: {offset} == {end_index}" + ) + elif offset >= end_index: raise ValueError( - f"Offset exceeds reachable characters or EOL in source line {parser_pos.line}: {parser_pos.offset} > {last_index}" + f"Offset exceeds reachable characters of line: {line}: {offset} >= {end_index}" ) - elif parser_pos.offset < 0: - raise ValueError("Unreachable offset, must be >= 0.") + return ParserPosition(line=line, offset=offset, eol=eol, eof=eof) - def translate(self, parser_pos: LinePosition) -> PartPosition: - self.validate(parser_pos) - part_pos = parser_pos_to_part_pos(self.source_text_parts, parser_pos) + def translate(self, pos: LinePosition) -> PartPosition: + parser_pos = self.validate_raw_parser_pos(pos) + part_pos = parser_pos_to_part_pos( + self.source_text_parts, parser_pos, self.line_to_part_pos + ) validate_part_position(part_pos) return part_pos def parser_pos_to_part_pos( parts: tuple[str, ...], - parser_pos: LinePosition, + parser_pos: ParserPosition, + line_to_part_pos: dict[int, PartPosition], ) -> PartPosition: """ Translate the given parser position into a template part position. - - Iterate over the template parts. - - Track the current line and offset while advancing into each part. + - Jump to the precomputed part position for the given line. + - Iterate over the subsequent template parts. + - Track the offset while advancing into each part. - When we reach the parser position then return the current part and the current offset from the start of that part. """ - pos = MutableLinePosition() + pos = MutableLinePosition(line=parser_pos.line, offset=0) + part_pos = line_to_part_pos[parser_pos.line] + start_text = parts[part_pos.index][part_pos.offset :] last_index = len(parts) - 1 - for index, part_text in enumerate(parts): - nls_found = part_text.count("\n") - if parser_pos.line > pos.line: # need more lines - nls_need = parser_pos.line - pos.line # how many are needed? - if nls_found >= nls_need: - pos.line += nls_need - lines_found = part_text.split("\n") - offset_found = len(lines_found[nls_need]) - if offset_found >= parser_pos.offset: - # needed lines, found lines, found offset - pos.offset = parser_pos.offset - total_offset = ( - sum(len(line) + 1 for line in lines_found[:nls_need]) - + parser_pos.offset - ) - return PartPosition(index, total_offset) - else: - # got enough lines, still need more offset - pos.offset = offset_found - elif nls_found > 0: - # some lines but still need more lines - last_nl_index = part_text.rfind("\n") - pos.line += nls_found - pos.offset = len(part_text[last_nl_index + 1 :]) - else: - # no lines, still need more lines - pos.offset += len(part_text) - elif parser_pos.line == pos.line: - # got enough lines, we just need more offset - first_nl_index = part_text.find("\n") - offset_found = ( - len(part_text[:first_nl_index]) if nls_found else len(part_text) - ) - offset_need = parser_pos.offset - pos.offset - if offset_found > offset_need: - pos.offset += offset_need - total_offset = offset_need - # had lines, found offset - return PartPosition(index, total_offset) - elif offset_found == offset_need: - if index != last_index and first_nl_index == -1: + for index, part_text in enumerate( + (start_text, *parts[part_pos.index + 1 :]), start=part_pos.index + ): + # got enough lines, we just need more offset + first_nl_index = part_text.find("\n") + offset_found = ( + len(part_text[:first_nl_index]) if first_nl_index != -1 else len(part_text) + ) + offset_need = parser_pos.offset - pos.offset + part_offset = 0 if index != part_pos.index else part_pos.offset + if offset_found > offset_need: + pos.offset += offset_need + part_offset += offset_need + return PartPosition(index, part_offset) + elif offset_found == offset_need: + part_offset += offset_need + if first_nl_index == -1: + if index != last_index: return PartPosition(index + 1, 0) + elif parser_pos.eof: # index is last_index + return PartPosition(index, part_offset) else: - return PartPosition(last_index, offset_found) + # This is the last index, a string, + # and the parser position is pointing off the end. + raise ValueError( + "Configured parser position lands at EOF but eof is False." + ) else: - pos.offset += offset_found + if parser_pos.eol: + return PartPosition(index, part_offset) + else: + raise ValueError( + "Configured parser position lands at EOL but eol is False." + ) else: - # We should have dropped out and failed earlier this would be a bug. - raise AssertionError( - f"Unexpected line: {pos.line} greater than asked for {parser_pos.line}" - ) + pos.offset += offset_found raise AssertionError( - "Unexpected position {pos}, did not reach required position {parser_pos}" + f"Unexpected position {pos}, did not reach required position {parser_pos}" ) diff --git a/tdom/parser_utils_test.py b/tdom/parser_utils_test.py index 87f258db..a54b7501 100644 --- a/tdom/parser_utils_test.py +++ b/tdom/parser_utils_test.py @@ -22,7 +22,7 @@ class TestParserPositionTranslator: def test_case_nontailing_string_ends_with_newline(self, ph_config): ppt = make_ppt(t"a\n{0}b", ph_config) assert ppt.translate(LinePosition(line=2, offset=0)) == PartPosition( - index=0, offset=2 + index=1, offset=0 ), """This could also be considered PartPosition(index=1, offset=0) but either way should work. """ assert ppt.translate( @@ -86,6 +86,12 @@ def test_offset_with_line(self, ph_config): # only 0, 1 and 2 are valid offsets for line 1 _ = ppt.translate(LinePosition(line=1, offset=3)) + def test_offset_with_line_in_middle_part(self, ph_config): + ppt = make_ppt(t"a\nb{0}cd\ne{1}\nfe", ph_config) + assert ppt.translate( + LinePosition(line=2, offset=1 + len(ph_config.make_placeholder(0)) + 2) + ) == PartPosition(index=2, offset=2) + def test_empty_strings(self, ph_config): ppt = make_ppt(t"{0}", ph_config) assert ppt.translate(LinePosition(line=1, offset=0)) == PartPosition(